You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/10/28 08:05:33 UTC
[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 360 - Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/360/
1 tests failed.
FAILED: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
Error Message:
Error from server at http://127.0.0.1:46525/u/h/control_collection: ERROR adding document SolrInputDocument(fields: [id=100006401, text_t=some text so that it not's negligent work to parse this doc, even though it's still a pretty short doc, _version_=1615549032016379904])
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46525/u/h/control_collection: ERROR adding document SolrInputDocument(fields: [id=100006401, text_t=some text so that it not's negligent work to parse this doc, even though it's still a pretty short doc, _version_=1615549032016379904])
at __randomizedtesting.SeedInfo.seed([98C8A0C53E10E698:109C9F1F90EC8B60]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:177)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:156)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.index_specific(AbstractFullDistribZkTestBase.java:879)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testConcurrentIndexing(FullSolrCloudDistribCmdsTest.java:697)
at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:139)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 14859 lines...]
[junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/init-core-data-001
[junit4] 2> 3573707 WARN (SUITE-FullSolrCloudDistribCmdsTest-seed#[98C8A0C53E10E698]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
[junit4] 2> 3573707 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[98C8A0C53E10E698]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 3573725 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[98C8A0C53E10E698]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 3573725 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[98C8A0C53E10E698]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3573725 INFO (SUITE-FullSolrCloudDistribCmdsTest-seed#[98C8A0C53E10E698]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /u/h
[junit4] 2> 3573727 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3573738 INFO (Thread-12301) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3573738 INFO (Thread-12301) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3573751 ERROR (Thread-12301) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3573838 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkTestServer start zk server on port:34988
[junit4] 2> 3573847 INFO (zkConnectionManagerCallback-4737-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3573888 INFO (zkConnectionManagerCallback-4739-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3573892 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3573893 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 3573894 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3573895 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3573916 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3573917 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3573918 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3573919 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3573920 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3573933 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3573934 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3573935 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 3574519 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3574521 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3574521 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3574521 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3574523 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@59c28f21{/u/h,null,AVAILABLE}
[junit4] 2> 3574524 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4d20d38a{HTTP/1.1,[http/1.1]}{127.0.0.1:46525}
[junit4] 2> 3574524 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.Server Started @3574979ms
[junit4] 2> 3574524 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/tempDir-001/control/data, hostContext=/u/h, hostPort=46525, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/control-001/cores}
[junit4] 2> 3574626 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3574626 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3574626 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3574626 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3574626 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3574626 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-28T06:19:01.711Z
[junit4] 2> 3574628 INFO (zkConnectionManagerCallback-4741-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3574629 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3574629 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/control-001/solr.xml
[junit4] 2> 3574632 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3574632 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3574654 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3576353 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34988/solr
[junit4] 2> 3576380 INFO (zkConnectionManagerCallback-4745-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3576424 INFO (zkConnectionManagerCallback-4747-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3576954 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46525_u%2Fh
[junit4] 2> 3576955 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.Overseer Overseer (id=73567867341897732-127.0.0.1:46525_u%2Fh-n_0000000000) starting
[junit4] 2> 3577002 INFO (zkConnectionManagerCallback-4754-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3577015 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34988/solr ready
[junit4] 2> 3577030 INFO (OverseerStateUpdate-73567867341897732-127.0.0.1:46525_u%2Fh-n_0000000000) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:46525_u%2Fh
[junit4] 2> 3577032 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46525_u%2Fh
[junit4] 2> 3577188 INFO (zkCallback-4753-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3577201 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3577273 INFO (OverseerStateUpdate-73567867341897732-127.0.0.1:46525_u%2Fh-n_0000000000) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3577276 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3577343 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3577343 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3577345 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/control-001/cores
[junit4] 2> 3577455 INFO (zkConnectionManagerCallback-4760-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3577473 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3577474 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34988/solr ready
[junit4] 2> 3577475 INFO (qtp881846889-25258) [n:127.0.0.1:46525_u%2Fh ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46525_u%252Fh&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3577490 INFO (OverseerThreadFactory-6154-thread-1-processing-n:127.0.0.1:46525_u%2Fh) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 3577601 INFO (qtp881846889-25253) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3577636 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3577636 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3578811 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 3578883 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 3579336 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 3579393 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3579394 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3579394 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 3579409 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=80.025390625, floorSegmentMB=2.0947265625, forceMergeDeletesPctAllowed=6.152401074499108, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=41.49410604339717
[junit4] 2> 3579432 WARN (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3579702 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3579702 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3579703 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3579703 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3579704 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3579706 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@293892f4[control_collection_shard1_replica_n1] main]
[junit4] 2> 3579707 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3579707 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3579707 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3579708 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615548956586016768
[junit4] 2> 3579718 INFO (searcherExecutor-6159-thread-1-processing-n:127.0.0.1:46525_u%2Fh x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@293892f4[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3579733 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 3579748 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3579748 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3579748 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46525/u/h/control_collection_shard1_replica_n1/
[junit4] 2> 3579748 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3579749 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:46525/u/h/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 3579749 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3579768 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46525/u/h/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 3579836 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3579838 INFO (qtp881846889-25254) [n:127.0.0.1:46525_u%2Fh c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2202
[junit4] 2> 3579870 INFO (qtp881846889-25258) [n:127.0.0.1:46525_u%2Fh ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3579949 INFO (zkCallback-4746-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3580879 INFO (qtp881846889-25258) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46525_u%252Fh&wt=javabin&version=2} status=0 QTime=3404
[junit4] 2> 3580924 INFO (zkConnectionManagerCallback-4765-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3580929 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3580929 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34988/solr ready
[junit4] 2> 3580929 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3580942 INFO (qtp881846889-25253) [n:127.0.0.1:46525_u%2Fh ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3581012 INFO (OverseerCollectionConfigSetProcessor-73567867341897732-127.0.0.1:46525_u%2Fh-n_0000000000) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3581010 INFO (OverseerThreadFactory-6154-thread-2-processing-n:127.0.0.1:46525_u%2Fh) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3581347 WARN (OverseerThreadFactory-6154-thread-2-processing-n:127.0.0.1:46525_u%2Fh) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 3581364 INFO (qtp881846889-25253) [n:127.0.0.1:46525_u%2Fh ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3581365 INFO (qtp881846889-25253) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=423
[junit4] 2> 3582156 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-1-001 of type NRT
[junit4] 2> 3582157 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3582215 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3582216 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3582216 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 3582265 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7f607861{/u/h,null,AVAILABLE}
[junit4] 2> 3582265 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5383e5dd{HTTP/1.1,[http/1.1]}{127.0.0.1:39330}
[junit4] 2> 3582265 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.Server Started @3582720ms
[junit4] 2> 3582265 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/u/h, hostPort=39330, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-1-001/cores}
[junit4] 2> 3582266 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3582266 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3582266 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3582266 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3582266 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3582266 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-28T06:19:09.351Z
[junit4] 2> 3582272 INFO (zkConnectionManagerCallback-4767-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3582302 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3582302 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-1-001/solr.xml
[junit4] 2> 3582325 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3582325 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3582338 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3582985 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34988/solr
[junit4] 2> 3582986 INFO (zkConnectionManagerCallback-4771-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3583005 INFO (zkConnectionManagerCallback-4773-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3583033 INFO (OverseerCollectionConfigSetProcessor-73567867341897732-127.0.0.1:46525_u%2Fh-n_0000000000) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3583049 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3583052 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3583052 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39330_u%2Fh
[junit4] 2> 3583086 INFO (zkCallback-4746-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3583086 INFO (zkCallback-4753-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3583126 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3583126 INFO (zkCallback-4764-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3583406 INFO (zkConnectionManagerCallback-4780-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3583411 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3583412 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34988/solr ready
[junit4] 2> 3583443 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3583538 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3583576 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3583577 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3583591 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-1-001/cores
[junit4] 2> 3583707 INFO (qtp1767149015-25312) [n:127.0.0.1:39330_u%2Fh ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39330_u%252Fh&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3583797 INFO (qtp1767149015-25307) [n:127.0.0.1:39330_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3583887 INFO (qtp881846889-25259) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=54
[junit4] 2> 3583889 INFO (qtp881846889-25260) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3583903 INFO (OverseerThreadFactory-6154-thread-3-processing-n:127.0.0.1:46525_u%2Fh) [n:127.0.0.1:46525_u%2Fh c:collection1 s:shard2 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:39330_u%2Fh for creating new replica of shard shard2 for collection collection1
[junit4] 2> 3583968 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh x:collection1_shard2_replica_n61] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3585128 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 3585232 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema [collection1_shard2_replica_n61] Schema name=test
[junit4] 2> 3586306 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 3586397 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n61' using configuration from collection collection1, trusted=true
[junit4] 2> 3586398 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n61' (registry 'solr.core.collection1.shard2.replica_n61') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3586398 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [[collection1_shard2_replica_n61] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-1-001/cores/collection1_shard2_replica_n61], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-1-001/cores/collection1_shard2_replica_n61/data/]
[junit4] 2> 3586439 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=80.025390625, floorSegmentMB=2.0947265625, forceMergeDeletesPctAllowed=6.152401074499108, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=41.49410604339717
[junit4] 2> 3586491 WARN (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3586812 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3586812 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3586814 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3586814 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3586830 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3586849 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.SolrIndexSearcher Opening [Searcher@1ff32369[collection1_shard2_replica_n61] main]
[junit4] 2> 3586864 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3586864 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3586865 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3586866 INFO (searcherExecutor-6173-thread-1-processing-n:127.0.0.1:39330_u%2Fh x:collection1_shard2_replica_n61 c:collection1 s:shard2) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [collection1_shard2_replica_n61] Registered new searcher Searcher@1ff32369[collection1_shard2_replica_n61] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3586866 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615548964091723776
[junit4] 2> 3586976 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node62=0}, version=0}
[junit4] 2> 3586991 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3586991 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3586991 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:39330/u/h/collection1_shard2_replica_n61/
[junit4] 2> 3586991 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3586991 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy http://127.0.0.1:39330/u/h/collection1_shard2_replica_n61/ has no replicas
[junit4] 2> 3586991 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3587026 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:39330/u/h/collection1_shard2_replica_n61/ shard2
[junit4] 2> 3587096 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3587098 INFO (qtp1767149015-25308) [n:127.0.0.1:39330_u%2Fh c:collection1 s:shard2 x:collection1_shard2_replica_n61] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=3129
[junit4] 2> 3587181 INFO (qtp1767149015-25312) [n:127.0.0.1:39330_u%2Fh c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:39330_u%252Fh&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=3474
[junit4] 2> 3587254 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3587850 INFO (OverseerCollectionConfigSetProcessor-73567867341897732-127.0.0.1:46525_u%2Fh-n_0000000000) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3588180 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-2-001 of type NRT
[junit4] 2> 3588181 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3588223 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3588223 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3588223 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3588242 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5624b48a{/u/h,null,AVAILABLE}
[junit4] 2> 3588242 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7c0e5fe8{HTTP/1.1,[http/1.1]}{127.0.0.1:37430}
[junit4] 2> 3588242 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.Server Started @3588697ms
[junit4] 2> 3588242 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/u/h, hostPort=37430, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-2-001/cores}
[junit4] 2> 3588242 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3588250 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3588250 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3588250 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3588250 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3588250 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-28T06:19:15.335Z
[junit4] 2> 3588252 INFO (zkConnectionManagerCallback-4783-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3588270 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3588270 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1055d95e7cc000c, likely client has closed socket
[junit4] 2> 3588270 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-2-001/solr.xml
[junit4] 2> 3588273 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3588273 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3588307 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3589117 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34988/solr
[junit4] 2> 3589130 INFO (zkConnectionManagerCallback-4787-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589134 INFO (zkConnectionManagerCallback-4789-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589159 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3589162 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3589162 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37430_u%2Fh
[junit4] 2> 3589172 INFO (zkCallback-4746-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3589172 INFO (zkCallback-4753-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3589172 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3589192 INFO (zkCallback-4764-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3589285 INFO (zkCallback-4788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3589301 INFO (zkCallback-4779-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3589411 INFO (zkConnectionManagerCallback-4796-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589464 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3589464 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34988/solr ready
[junit4] 2> 3589465 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3589552 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3589620 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3589620 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3589635 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-2-001/cores
[junit4] 2> 3590014 INFO (qtp855049329-25349) [n:127.0.0.1:37430_u%2Fh ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:37430_u%252Fh&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3590062 INFO (qtp855049329-25344) [n:127.0.0.1:37430_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3590088 INFO (qtp1767149015-25307) [n:127.0.0.1:39330_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard2.replica_n61:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 3590122 INFO (qtp1767149015-25314) [n:127.0.0.1:39330_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=33
[junit4] 2> 3590134 INFO (qtp881846889-25260) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 3590136 INFO (qtp881846889-25259) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3590138 INFO (OverseerThreadFactory-6154-thread-4-processing-n:127.0.0.1:46525_u%2Fh) [n:127.0.0.1:46525_u%2Fh c:collection1 s:shard3 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:37430_u%2Fh for creating new replica of shard shard3 for collection collection1
[junit4] 2> 3590224 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh x:collection1_shard3_replica_n63] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n63&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3590240 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3591270 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 3591323 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.IndexSchema [collection1_shard3_replica_n63] Schema name=test
[junit4] 2> 3591734 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 3591794 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard3_replica_n63' using configuration from collection collection1, trusted=true
[junit4] 2> 3591795 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard3.replica_n63' (registry 'solr.core.collection1.shard3.replica_n63') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3591807 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrCore [[collection1_shard3_replica_n63] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-2-001/cores/collection1_shard3_replica_n63], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-2-001/cores/collection1_shard3_replica_n63/data/]
[junit4] 2> 3591810 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=80.025390625, floorSegmentMB=2.0947265625, forceMergeDeletesPctAllowed=6.152401074499108, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=41.49410604339717
[junit4] 2> 3591861 WARN (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3592315 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3592315 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3592317 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3592317 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3592318 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3592319 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.SolrIndexSearcher Opening [Searcher@52fff9ef[collection1_shard3_replica_n63] main]
[junit4] 2> 3592388 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3592409 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3592409 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3592410 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615548969905029120
[junit4] 2> 3592413 INFO (searcherExecutor-6187-thread-1-processing-n:127.0.0.1:37430_u%2Fh x:collection1_shard3_replica_n63 c:collection1 s:shard3) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SolrCore [collection1_shard3_replica_n63] Registered new searcher Searcher@52fff9ef[collection1_shard3_replica_n63] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3592447 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard3 to Terms{values={core_node64=0}, version=0}
[junit4] 2> 3592449 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3592449 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3592449 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37430/u/h/collection1_shard3_replica_n63/
[junit4] 2> 3592449 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3592462 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy http://127.0.0.1:37430/u/h/collection1_shard3_replica_n63/ has no replicas
[junit4] 2> 3592462 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3592465 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37430/u/h/collection1_shard3_replica_n63/ shard3
[junit4] 2> 3592499 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3592501 INFO (qtp855049329-25345) [n:127.0.0.1:37430_u%2Fh c:collection1 s:shard3 x:collection1_shard3_replica_n63] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n63&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2277
[junit4] 2> 3592465 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3592560 INFO (qtp855049329-25349) [n:127.0.0.1:37430_u%2Fh c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:37430_u%252Fh&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2} status=0 QTime=2545
[junit4] 2> 3592608 INFO (zkCallback-4788-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3592632 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 3593341 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-3-001 of type NRT
[junit4] 2> 3593341 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3593342 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3593342 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3593342 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 3593343 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3adf75a3{/u/h,null,AVAILABLE}
[junit4] 2> 3593343 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3c7651fb{HTTP/1.1,[http/1.1]}{127.0.0.1:45808}
[junit4] 2> 3593343 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.e.j.s.Server Started @3593798ms
[junit4] 2> 3593376 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/u/h, hostPort=45808, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-3-001/cores}
[junit4] 2> 3593417 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3593418 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3593418 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3593418 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3593418 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3593418 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-28T06:19:20.503Z
[junit4] 2> 3593438 INFO (zkConnectionManagerCallback-4799-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593439 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3593439 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-3-001/solr.xml
[junit4] 2> 3593455 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3593455 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3593507 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3593979 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34988/solr
[junit4] 2> 3594002 INFO (zkConnectionManagerCallback-4803-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3594006 INFO (zkConnectionManagerCallback-4805-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3594022 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3594043 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3594043 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45808_u%2Fh
[junit4] 2> 3594044 INFO (zkCallback-4746-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3594044 INFO (zkCallback-4788-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3594054 INFO (zkCallback-4753-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3594054 INFO (zkCallback-4764-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3594054 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3594070 INFO (zkCallback-4779-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3594090 INFO (zkCallback-4804-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3594091 INFO (OverseerCollectionConfigSetProcessor-73567867341897732-127.0.0.1:46525_u%2Fh-n_0000000000) [n:127.0.0.1:46525_u%2Fh ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3594095 INFO (zkCallback-4795-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3594169 INFO (zkConnectionManagerCallback-4812-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3594170 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 3594171 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34988/solr ready
[junit4] 2> 3594184 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3594259 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3594310 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3594310 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3594312 INFO (TEST-FullSolrCloudDistribCmdsTest.test-seed#[98C8A0C53E10E698]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-3-001/cores
[junit4] 2> 3594507 INFO (qtp855049329-25348) [n:127.0.0.1:37430_u%2Fh ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:45808_u%252Fh&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3594545 INFO (qtp855049329-25344) [n:127.0.0.1:37430_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard3.replica_n63:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 3594546 INFO (qtp855049329-25344) [n:127.0.0.1:37430_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3594591 INFO (qtp1767149015-25313) [n:127.0.0.1:39330_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard2.replica_n61:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 3594592 INFO (qtp1767149015-25314) [n:127.0.0.1:39330_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3594603 INFO (qtp1680570307-25386) [n:127.0.0.1:45808_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3594605 INFO (qtp881846889-25259) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 3594635 INFO (qtp881846889-25260) [n:127.0.0.1:46525_u%2Fh ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=29
[junit4] 2> 3594644 INFO (OverseerThreadFactory-6154-thread-5-processing-n:127.0.0.1:46525_u%2Fh) [n:127.0.0.1:46525_u%2Fh c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:45808_u%2Fh for creating new replica of shard shard1 for collection collection1
[junit4] 2> 3594665 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh x:collection1_shard1_replica_n65] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n65&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3594682 INFO (zkCallback-4788-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 3594692 INFO (zkCallback-4772-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
[junit4] 2> 3594754 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 3594921 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.IndexSchema [collection1_shard1_replica_n65] Schema name=test
[junit4] 2> 3595211 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 3595281 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n65' using configuration from collection collection1, trusted=true
[junit4] 2> 3595282 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n65' (registry 'solr.core.collection1.shard1.replica_n65') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5634a4dd
[junit4] 2> 3595302 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.SolrCore [[collection1_shard1_replica_n65] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-3-001/cores/collection1_shard1_replica_n65], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_98C8A0C53E10E698-001/shard-3-001/cores/collection1_shard1_replica_n65/data/]
[junit4] 2> 3595304 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=80.025390625, floorSegmentMB=2.0947265625, forceMergeDeletesPctAllowed=6.152401074499108, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=41.49410604339717
[junit4] 2> 3595355 WARN (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3595732 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3595732 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3595734 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3595734 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3595735 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 3595736 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x:collection1_shard1_replica_n65] o.a.s.s.SolrIndexSearcher Opening [Searcher@65106d4b[collection1_shard1_replica_n65] main]
[junit4] 2> 3595777 INFO (qtp1680570307-25381) [n:127.0.0.1:45808_u%2Fh c:collection1 s:shard1 x
[...truncated too long message...]
s.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1474879565
[copy] Copying 241 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1474879565
[delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1474879565
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 30ms :: artifacts dl 4ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 512 minutes 40 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 363 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/363/
1 tests failed.
FAILED: org.apache.solr.cloud.TestCloudPivotFacet.test
Error Message:
{main(facet=true&facet.pivot={!stats%3Dst0}bogus_not_in_any_doc_s&facet.pivot={!stats%3Dst2}pivot_z_s1,pivot_x_s,pivot_tdt&facet.limit=15&facet.missing=true&facet.sort=index&facet.overrequest.count=0),extra(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index)} ==> Stddev of sk2 => pivot_z_s1,pivot_x_s,pivot_tdt: {params(rows=0),defaults({main({main(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index),extra(fq={!term+f%3Dpivot_z_s1}a)}),extra(fq={!term+f%3Dpivot_x_s}b)})} expected:<0.0> but was:<20724.30287367949>
Stack Trace:
java.lang.AssertionError: {main(facet=true&facet.pivot={!stats%3Dst0}bogus_not_in_any_doc_s&facet.pivot={!stats%3Dst2}pivot_z_s1,pivot_x_s,pivot_tdt&facet.limit=15&facet.missing=true&facet.sort=index&facet.overrequest.count=0),extra(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index)} ==> Stddev of sk2 => pivot_z_s1,pivot_x_s,pivot_tdt: {params(rows=0),defaults({main({main(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index),extra(fq={!term+f%3Dpivot_z_s1}a)}),extra(fq={!term+f%3Dpivot_x_s}b)})} expected:<0.0> but was:<20724.30287367949>
at __randomizedtesting.SeedInfo.seed([E8745FA6D96CE260:6020607C77908F98]:0)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:292)
at org.apache.solr.cloud.TestCloudPivotFacet.test(TestCloudPivotFacet.java:239)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: Stddev of sk2 => pivot_z_s1,pivot_x_s,pivot_tdt: {params(rows=0),defaults({main({main(rows=0&q=*:*&stats=true&stats.field={!key%3Dsk1+tag%3Dst1,st2}pivot_d&stats.field={!key%3Dsk2+tag%3Dst2,st3}pivot_dt1&stats.field={!key%3Dsk3+tag%3Dst3,st4}pivot_z_s&_test_miss=true&_test_sort=index),extra(fq={!term+f%3Dpivot_z_s1}a)}),extra(fq={!term+f%3Dpivot_x_s}b)})} expected:<0.0> but was:<20724.30287367949>
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:443)
at org.apache.solr.cloud.TestCloudPivotFacet.assertNumerics(TestCloudPivotFacet.java:743)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotStats(TestCloudPivotFacet.java:402)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotData(TestCloudPivotFacet.java:350)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:313)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:324)
at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:282)
... 42 more
Build Log:
[...truncated 14318 lines...]
[junit4] Suite: org.apache.solr.cloud.TestCloudPivotFacet
[junit4] 2> 1969745 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/init-core-data-001
[junit4] 2> 1969746 WARN (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
[junit4] 2> 1969746 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1969748 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 1969748 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1969749 INFO (SUITE-TestCloudPivotFacet-seed#[E8745FA6D96CE260]-worker) [ ] o.a.s.c.TestCloudPivotFacet init'ing useFieldRandomizedFactor = 4
[junit4] 2> 1969755 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1969755 INFO (Thread-3174) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1969755 INFO (Thread-3174) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1969757 ERROR (Thread-3174) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1969855 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkTestServer start zk server on port:43751
[junit4] 2> 1969859 INFO (zkConnectionManagerCallback-3792-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1969864 INFO (zkConnectionManagerCallback-3794-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1969872 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1969874 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1969876 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1969878 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1969879 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1969881 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1969882 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1969884 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1969885 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1969887 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1969888 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1969890 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1970067 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 1970069 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1970069 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1970069 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1970069 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@69a7986a{/,null,AVAILABLE}
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5c0d69{HTTP/1.1,[http/1.1]}{127.0.0.1:46634}
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server Started @1970163ms
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/tempDir-001/control/data, hostContext=/, hostPort=46634, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/cores}
[junit4] 2> 1970071 ERROR (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1970071 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 1970072 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1970072 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1970072 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-31T09:03:58.705Z
[junit4] 2> 1970074 INFO (zkConnectionManagerCallback-3796-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970075 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1970075 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/solr.xml
[junit4] 2> 1970080 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1970080 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1970082 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1970202 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43751/solr
[junit4] 2> 1970203 INFO (zkConnectionManagerCallback-3800-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970207 INFO (zkConnectionManagerCallback-3802-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970304 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46634_
[junit4] 2> 1970305 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.Overseer Overseer (id=72195968331743236-127.0.0.1:46634_-n_0000000000) starting
[junit4] 2> 1970313 INFO (zkConnectionManagerCallback-3809-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970317 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1970319 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46634_
[junit4] 2> 1970319 INFO (OverseerStateUpdate-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:46634_
[junit4] 2> 1970336 INFO (OverseerStateUpdate-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1970341 INFO (zkCallback-3808-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1970358 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1970386 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1970403 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1970403 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1970405 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [n:127.0.0.1:46634_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/cores
[junit4] 2> 1970438 INFO (zkConnectionManagerCallback-3815-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1970439 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1970440 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1970442 INFO (qtp2043184068-14033) [n:127.0.0.1:46634_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46634_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1970446 INFO (OverseerThreadFactory-5492-thread-1-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1970556 INFO (qtp2043184068-14028) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1970564 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1970564 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1971608 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 1971635 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1971828 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1971853 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1971854 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1971854 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1971859 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=35.2265625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=14.107885589266665, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=27.694887829024218
[junit4] 2> 1971865 WARN (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1971950 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1971950 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1971952 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1971952 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1971955 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=0.7294921875, floorSegmentMB=0.921875, forceMergeDeletesPctAllowed=7.396377397787985, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7317311934972269, deletesPctAllowed=48.98606555317808
[junit4] 2> 1971957 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@352f4d3d[control_collection_shard1_replica_n1] main]
[junit4] 2> 1971959 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1971959 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1971960 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1971962 INFO (searcherExecutor-5497-thread-1-processing-n:127.0.0.1:46634_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@352f4d3d[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1971968 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615831121894834176
[junit4] 2> 1971976 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 1971979 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1971979 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1971980 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46634/control_collection_shard1_replica_n1/
[junit4] 2> 1971980 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1971980 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:46634/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1971980 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1971984 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46634/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1972139 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1972142 INFO (qtp2043184068-14029) [n:127.0.0.1:46634_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1578
[junit4] 2> 1972145 INFO (qtp2043184068-14033) [n:127.0.0.1:46634_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1972242 INFO (zkCallback-3801-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1972449 INFO (OverseerCollectionConfigSetProcessor-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1973145 INFO (qtp2043184068-14033) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:46634_&wt=javabin&version=2} status=0 QTime=2703
[junit4] 2> 1973152 INFO (zkConnectionManagerCallback-3820-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973153 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1973154 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1973154 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1973155 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1973158 INFO (OverseerThreadFactory-5492-thread-2-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1973364 WARN (OverseerThreadFactory-5492-thread-2-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1973367 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1973367 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=211
[junit4] 2> 1973549 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001 of type NRT
[junit4] 2> 1973550 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 1973551 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1973551 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1973551 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1973552 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4e05206a{/,null,AVAILABLE}
[junit4] 2> 1973553 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6fc5b35c{HTTP/1.1,[http/1.1]}{127.0.0.1:33700}
[junit4] 2> 1973553 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server Started @1973646ms
[junit4] 2> 1973553 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=33700, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/cores}
[junit4] 2> 1973554 ERROR (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1973554 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-31T09:04:02.187Z
[junit4] 2> 1973556 INFO (zkConnectionManagerCallback-3822-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973557 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1973557 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/solr.xml
[junit4] 2> 1973562 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1973562 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1973564 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1973928 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43751/solr
[junit4] 2> 1973930 INFO (zkConnectionManagerCallback-3826-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973932 INFO (zkConnectionManagerCallback-3828-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973941 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1973946 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1973946 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33700_
[junit4] 2> 1973947 INFO (zkCallback-3808-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1973947 INFO (zkCallback-3801-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1973947 INFO (zkCallback-3819-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1973948 INFO (zkCallback-3827-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1973977 INFO (zkConnectionManagerCallback-3835-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1973978 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1973979 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1973980 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1974009 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1974027 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1974027 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1974030 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/cores
[junit4] 2> 1974073 INFO (qtp502179821-14088) [n:127.0.0.1:33700_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33700_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1974076 INFO (OverseerCollectionConfigSetProcessor-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1974081 INFO (qtp502179821-14083) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=2
[junit4] 2> 1974085 INFO (qtp2043184068-14028) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=2
[junit4] 2> 1974086 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 1974087 INFO (OverseerThreadFactory-5492-thread-3-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ c:collection1 s:shard2 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:33700_ for creating new replica of shard shard2 for collection collection1
[junit4] 2> 1974092 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ x:collection1_shard2_replica_n41] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1975112 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 1975129 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
[junit4] 2> 1975280 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1975298 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
[junit4] 2> 1975299 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1975299 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
[junit4] 2> 1975306 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=35.2265625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=14.107885589266665, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=27.694887829024218
[junit4] 2> 1975310 WARN (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1975383 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1975383 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1975385 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1975385 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1975387 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=0.7294921875, floorSegmentMB=0.921875, forceMergeDeletesPctAllowed=7.396377397787985, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7317311934972269, deletesPctAllowed=48.98606555317808
[junit4] 2> 1975388 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f1ba90e[collection1_shard2_replica_n41] main]
[junit4] 2> 1975389 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1975390 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1975391 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1975392 INFO (searcherExecutor-5511-thread-1-processing-n:127.0.0.1:33700_ x:collection1_shard2_replica_n41 c:collection1 s:shard2) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@3f1ba90e[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1975392 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615831125485158400
[junit4] 2> 1975399 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node42=0}, version=0}
[junit4] 2> 1975402 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1975402 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1975402 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:33700/collection1_shard2_replica_n41/
[junit4] 2> 1975402 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1975403 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:33700/collection1_shard2_replica_n41/ has no replicas
[junit4] 2> 1975403 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1975407 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:33700/collection1_shard2_replica_n41/ shard2
[junit4] 2> 1975559 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1975562 INFO (qtp502179821-14084) [n:127.0.0.1:33700_ c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1471
[junit4] 2> 1975566 INFO (qtp502179821-14088) [n:127.0.0.1:33700_ c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:33700_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1492
[junit4] 2> 1975753 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001 of type NRT
[junit4] 2> 1975755 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 1975757 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1975757 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1975757 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1975758 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7bfd2e5c{/,null,AVAILABLE}
[junit4] 2> 1975760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4947e03d{HTTP/1.1,[http/1.1]}{127.0.0.1:37293}
[junit4] 2> 1975760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server Started @1975853ms
[junit4] 2> 1975760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=37293, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/cores}
[junit4] 2> 1975760 ERROR (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1975761 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-31T09:04:04.394Z
[junit4] 2> 1975763 INFO (zkConnectionManagerCallback-3838-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1975764 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1975764 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/solr.xml
[junit4] 2> 1975769 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1975769 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1975771 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1975977 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43751/solr
[junit4] 2> 1975979 INFO (zkConnectionManagerCallback-3842-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1975982 INFO (zkConnectionManagerCallback-3844-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1975989 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1975994 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1975994 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37293_
[junit4] 2> 1975996 INFO (zkCallback-3834-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975996 INFO (zkCallback-3808-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975996 INFO (zkCallback-3819-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975996 INFO (zkCallback-3801-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975996 INFO (zkCallback-3827-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1975997 INFO (zkCallback-3843-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1976025 INFO (zkConnectionManagerCallback-3851-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1976026 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1976027 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1976028 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1976057 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1976075 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1976075 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1976078 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/cores
[junit4] 2> 1976079 INFO (OverseerCollectionConfigSetProcessor-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1976134 INFO (qtp502179821-14090) [n:127.0.0.1:33700_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:37293_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1976140 INFO (qtp502179821-14087) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard2.replica_n41:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 1976142 INFO (qtp502179821-14083) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1976144 INFO (qtp490614089-14125) [n:127.0.0.1:37293_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1976145 INFO (qtp2043184068-14028) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1976146 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1976146 INFO (OverseerThreadFactory-5492-thread-4-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:37293_ for creating new replica of shard shard1 for collection collection1
[junit4] 2> 1976149 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ x:collection1_shard1_replica_n43] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1977168 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 1977186 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema [collection1_shard1_replica_n43] Schema name=test
[junit4] 2> 1977328 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1977351 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n43' using configuration from collection collection1, trusted=true
[junit4] 2> 1977352 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n43' (registry 'solr.core.collection1.shard1.replica_n43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1977353 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [[collection1_shard1_replica_n43] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/cores/collection1_shard1_replica_n43], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-2-001/cores/collection1_shard1_replica_n43/data/]
[junit4] 2> 1977358 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=35.2265625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=14.107885589266665, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=27.694887829024218
[junit4] 2> 1977362 WARN (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1977434 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1977434 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1977436 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1977436 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1977438 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=0.7294921875, floorSegmentMB=0.921875, forceMergeDeletesPctAllowed=7.396377397787985, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7317311934972269, deletesPctAllowed=48.98606555317808
[junit4] 2> 1977439 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.SolrIndexSearcher Opening [Searcher@35041ba[collection1_shard1_replica_n43] main]
[junit4] 2> 1977440 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1977441 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1977442 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1977442 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615831127634739200
[junit4] 2> 1977443 INFO (searcherExecutor-5525-thread-1-processing-n:127.0.0.1:37293_ x:collection1_shard1_replica_n43 c:collection1 s:shard1) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SolrCore [collection1_shard1_replica_n43] Registered new searcher Searcher@35041ba[collection1_shard1_replica_n43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1977447 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node44=0}, version=0}
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37293/collection1_shard1_replica_n43/
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.SyncStrategy http://127.0.0.1:37293/collection1_shard1_replica_n43/ has no replicas
[junit4] 2> 1977450 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1977454 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37293/collection1_shard1_replica_n43/ shard1
[junit4] 2> 1977557 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1977562 INFO (qtp490614089-14120) [n:127.0.0.1:37293_ c:collection1 s:shard1 x:collection1_shard1_replica_n43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1412
[junit4] 2> 1977564 INFO (qtp502179821-14090) [n:127.0.0.1:33700_ c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:37293_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1430
[junit4] 2> 1977745 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001 of type NRT
[junit4] 2> 1977746 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 1977748 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1977748 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1977748 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1977748 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6489f990{/,null,AVAILABLE}
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6b364fcb{HTTP/1.1,[http/1.1]}{127.0.0.1:38180}
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.e.j.s.Server Started @1977841ms
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=38180, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/cores}
[junit4] 2> 1977749 ERROR (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1977749 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-31T09:04:06.382Z
[junit4] 2> 1977754 INFO (zkConnectionManagerCallback-3854-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1977755 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1977755 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/solr.xml
[junit4] 2> 1977760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1977760 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1977762 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1977878 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43751/solr
[junit4] 2> 1977880 INFO (zkConnectionManagerCallback-3858-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1977883 INFO (zkConnectionManagerCallback-3860-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1977890 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1977900 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1977900 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38180_
[junit4] 2> 1977901 INFO (zkCallback-3819-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3843-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3808-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3834-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3801-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977902 INFO (zkCallback-3850-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977903 INFO (zkCallback-3859-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977903 INFO (zkCallback-3827-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1977934 INFO (zkConnectionManagerCallback-3867-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1977935 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1977937 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:43751/solr ready
[junit4] 2> 1977937 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1977967 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1977985 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1977986 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1977988 INFO (TEST-TestCloudPivotFacet.test-seed#[E8745FA6D96CE260]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/cores
[junit4] 2> 1978078 INFO (qtp502179821-14089) [n:127.0.0.1:33700_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:38180_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1978081 INFO (OverseerCollectionConfigSetProcessor-72195968331743236-127.0.0.1:46634_-n_0000000000) [n:127.0.0.1:46634_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1978084 INFO (qtp502179821-14087) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard2.replica_n41:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1978085 INFO (qtp502179821-14083) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1978088 INFO (qtp490614089-14121) [n:127.0.0.1:37293_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard1.replica_n43:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 1978089 INFO (qtp490614089-14127) [n:127.0.0.1:37293_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1978091 INFO (qtp643415226-14162) [n:127.0.0.1:38180_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1978092 INFO (qtp2043184068-14028) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1978093 INFO (qtp2043184068-14032) [n:127.0.0.1:46634_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 1978095 INFO (qtp502179821-14087) [n:127.0.0.1:33700_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard2.replica_n41:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1978096 INFO (OverseerThreadFactory-5492-thread-5-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_ c:collection1 s:shard2 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:38180_ for creating new replica of shard shard2 for collection collection1
[junit4] 2> 1978099 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ x:collection1_shard2_replica_n45] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1979125 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 1979141 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema [collection1_shard2_replica_n45] Schema name=test
[junit4] 2> 1979293 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1979314 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n45' using configuration from collection collection1, trusted=true
[junit4] 2> 1979323 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n45' (registry 'solr.core.collection1.shard2.replica_n45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@18fe091f
[junit4] 2> 1979324 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [[collection1_shard2_replica_n45] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/cores/collection1_shard2_replica_n45], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet_E8745FA6D96CE260-001/shard-3-001/cores/collection1_shard2_replica_n45/data/]
[junit4] 2> 1979329 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=35.2265625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=14.107885589266665, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0, deletesPctAllowed=27.694887829024218
[junit4] 2> 1979333 WARN (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1979405 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1979405 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1979407 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1979407 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1979409 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=9, maxMergedSegmentMB=0.7294921875, floorSegmentMB=0.921875, forceMergeDeletesPctAllowed=7.396377397787985, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7317311934972269, deletesPctAllowed=48.98606555317808
[junit4] 2> 1979410 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.SolrIndexSearcher Opening [Searcher@711642ac[collection1_shard2_replica_n45] main]
[junit4] 2> 1979412 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1979412 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1979413 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1979413 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615831129701482496
[junit4] 2> 1979415 INFO (searcherExecutor-5539-thread-1-processing-n:127.0.0.1:38180_ x:collection1_shard2_replica_n45 c:collection1 s:shard2) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.SolrCore [collection1_shard2_replica_n45] Registered new searcher Searcher@711642ac[collection1_shard2_replica_n45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1979422 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard2 to Terms{values={core_node42=0, core_node46=0}, version=1}
[junit4] 2> 1979424 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_n45
[junit4] 2> 1979424 INFO (updateExecutor-3855-thread-1-processing-n:127.0.0.1:38180_ x:collection1_shard2_replica_n45 c:collection1 s:shard2) [n:127.0.0.1:38180_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1979425 INFO (recoveryExecutor-3856-thread-1-processing-n:127.0.0.1:38180_ x:collection1_shard2_replica_n45 c:collection1 s:shard2 r:core_node46) [n:127.0.0.1:38180_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1979426 INFO (recoveryExecutor-3856-thread-1-processing-n:127.0.0.1:38180_ x:collection1_shard2_replica_n45 c:collection1 s:shard2 r:core_node46) [n:127.0.0.1:38180_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_n45] o.a.s.c.RecoveryStrategy startupVersions is empty
[junit4] 2> 1979427 INFO (qtp643415226-14157) [n:127.0.0.1:38180_ c:collection1 s:shard2 x:collection1_shard2_replica_n45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/co
[...truncated too long message...]
kout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1821347088
[copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1821347088
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1821347088
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 49ms :: artifacts dl 4ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 282 minutes 36 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 362 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/362/
3 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([949909B0C215D49:811DAF41A2DD30B1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:920)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1477)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:145)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest: 1) Thread[id=5723, name=searcherExecutor-2132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=5686, name=searcherExecutor-2118-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=5766, name=searcherExecutor-2146-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=5904, name=searcherExecutor-2180-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=5644, name=searcherExecutor-2104-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=5869, name=searcherExecutor-2166-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=5941, name=searcherExecutor-2194-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest:
1) Thread[id=5723, name=searcherExecutor-2132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=5686, name=searcherExecutor-2118-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=5766, name=searcherExecutor-2146-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=5904, name=searcherExecutor-2180-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=5644, name=searcherExecutor-2104-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=5869, name=searcherExecutor-2166-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=5941, name=searcherExecutor-2194-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([949909B0C215D49]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=5723, name=searcherExecutor-2132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=5686, name=searcherExecutor-2118-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=5766, name=searcherExecutor-2146-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=5904, name=searcherExecutor-2180-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=5644, name=searcherExecutor-2104-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=5869, name=searcherExecutor-2166-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=5941, name=searcherExecutor-2194-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=5723, name=searcherExecutor-2132-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=5686, name=searcherExecutor-2118-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=5766, name=searcherExecutor-2146-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=5904, name=searcherExecutor-2180-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=5644, name=searcherExecutor-2104-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=5869, name=searcherExecutor-2166-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=5941, name=searcherExecutor-2194-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([949909B0C215D49]:0)
Build Log:
[...truncated 13860 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/init-core-data-001
[junit4] 2> 779753 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=57 numCloses=57
[junit4] 2> 779753 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 779755 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 779755 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 779755 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_m/xy
[junit4] 2> 781737 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 784178 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 784871 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Logging to org.apache.logging.slf4j.Log4jLogger@3c686729 via org.mortbay.log.Slf4jLog
[junit4] 2> 784924 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 785806 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 785928 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_39760_hdfs____.gugpqj/webapp
[junit4] 2> 787522 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39760
[junit4] 2> 789386 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 789393 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 789420 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_38162_datanode____uc2b67/webapp
[junit4] 2> 789895 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38162
[junit4] 2> 791647 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 791648 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 791689 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_35321_datanode____1ri1h3/webapp
[junit4] 2> 792959 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[949909B0C215D49]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35321
[junit4] 2> 795242 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:39808) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 795243 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:39808) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 795962 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7173ecbb8897a: from storage DS-3a2efac9-5dc3-49a8-897c-356737a8a8b0 node DatanodeRegistration(127.0.0.1:44434, datanodeUuid=6c94df5d-b015-4457-96ef-c82bc84563ac, infoPort=40113, infoSecurePort=0, ipcPort=39705, storageInfo=lv=-56;cid=testClusterID;nsid=1544823971;c=0), blocks: 0, hasStaleStorage: true, processing time: 9 msecs
[junit4] 2> 795963 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7173ecbe66b90: from storage DS-4ddac0f4-88bd-4a15-abe5-c5925c828a6a node DatanodeRegistration(127.0.0.1:37765, datanodeUuid=56a425b1-8fa9-48b5-9907-4c713cae058b, infoPort=35248, infoSecurePort=0, ipcPort=45180, storageInfo=lv=-56;cid=testClusterID;nsid=1544823971;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 795963 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7173ecbb8897a: from storage DS-d14b8728-3e26-4b62-ad71-642792614414 node DatanodeRegistration(127.0.0.1:44434, datanodeUuid=6c94df5d-b015-4457-96ef-c82bc84563ac, infoPort=40113, infoSecurePort=0, ipcPort=39705, storageInfo=lv=-56;cid=testClusterID;nsid=1544823971;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 795969 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x7173ecbe66b90: from storage DS-672b6f93-849d-4a48-a7ea-4f9c043c05ea node DatanodeRegistration(127.0.0.1:37765, datanodeUuid=56a425b1-8fa9-48b5-9907-4c713cae058b, infoPort=35248, infoSecurePort=0, ipcPort=45180, storageInfo=lv=-56;cid=testClusterID;nsid=1544823971;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 797674 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 797674 INFO (Thread-1656) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 797674 INFO (Thread-1656) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 797677 ERROR (Thread-1656) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 797774 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkTestServer start zk server on port:41780
[junit4] 2> 797793 INFO (zkConnectionManagerCallback-1083-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 797813 INFO (zkConnectionManagerCallback-1085-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 797831 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 797839 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 797841 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 797843 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 797845 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 797848 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 797858 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 797865 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 797874 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 797882 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 797892 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 797896 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 798573 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 798575 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 798575 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 798575 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 798577 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4327f207{/_m/xy,null,AVAILABLE}
[junit4] 2> 798578 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@21c29571{HTTP/1.1,[http/1.1]}{127.0.0.1:37095}
[junit4] 2> 798578 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server Started @798773ms
[junit4] 2> 798578 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:39808/hdfs__localhost_39808__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001_tempDir-002_control_data, replicaType=NRT, hostContext=/_m/xy, hostPort=37095, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/cores}
[junit4] 2> 798579 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 798605 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 798605 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 798606 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 798606 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 798606 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-30T00:58:20.248Z
[junit4] 2> 798652 INFO (zkConnectionManagerCallback-1087-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 798663 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 798663 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/solr.xml
[junit4] 2> 798691 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 798691 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 798693 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 799227 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41780/solr
[junit4] 2> 799250 INFO (zkConnectionManagerCallback-1091-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 799255 INFO (zkConnectionManagerCallback-1093-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 799555 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37095__m%2Fxy
[junit4] 2> 799557 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.Overseer Overseer (id=72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) starting
[junit4] 2> 799635 INFO (zkConnectionManagerCallback-1100-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 799646 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 799656 INFO (OverseerStateUpdate-72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:37095__m%2Fxy
[junit4] 2> 799684 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37095__m%2Fxy
[junit4] 2> 799697 INFO (zkCallback-1092-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 799715 INFO (zkCallback-1099-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 799725 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 799777 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 799875 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 799875 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 799878 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/cores
[junit4] 2> 799933 INFO (zkConnectionManagerCallback-1106-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 799934 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 799935 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 799937 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37095__m%252Fxy&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 799955 INFO (OverseerThreadFactory-2099-thread-1-processing-n:127.0.0.1:37095__m%2Fxy) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 800193 INFO (qtp731027699-5599) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 800221 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 800222 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 801303 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 801430 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 803814 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 803875 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 803876 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 803877 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 803917 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=21.619140625, floorSegmentMB=0.26953125, forceMergeDeletesPctAllowed=22.12888844486297, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6395981875322819, deletesPctAllowed=45.80989708961823
[junit4] 2> 803959 WARN (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 804245 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 804245 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 804247 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 804247 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 804251 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=88.462890625, floorSegmentMB=1.7255859375, forceMergeDeletesPctAllowed=24.202588505252482, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6912498608789626, deletesPctAllowed=31.804024835151772
[junit4] 2> 804252 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@27e94675[control_collection_shard1_replica_n1] main]
[junit4] 2> 804260 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 804262 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 804264 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 804265 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615709974987538432
[junit4] 2> 804267 INFO (searcherExecutor-2104-thread-1-processing-n:127.0.0.1:37095__m%2Fxy x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@27e94675[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 804293 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 804316 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 804316 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 804316 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37095/_m/xy/control_collection_shard1_replica_n1/
[junit4] 2> 804317 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 804317 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:37095/_m/xy/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 804317 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 804325 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37095/_m/xy/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 804379 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 804383 INFO (qtp731027699-5600) [n:127.0.0.1:37095__m%2Fxy c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=4161
[junit4] 2> 804390 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 804482 INFO (zkCallback-1092-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 805390 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37095__m%252Fxy&wt=javabin&version=2} status=0 QTime=5453
[junit4] 2> 805423 INFO (zkConnectionManagerCallback-1111-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 805426 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 805427 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 805434 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 805437 INFO (qtp731027699-5599) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 805450 INFO (OverseerThreadFactory-2099-thread-2-processing-n:127.0.0.1:37095__m%2Fxy) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 805450 INFO (OverseerCollectionConfigSetProcessor-72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 805720 WARN (OverseerThreadFactory-2099-thread-2-processing-n:127.0.0.1:37095__m%2Fxy) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 805724 INFO (qtp731027699-5599) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 805724 INFO (qtp731027699-5599) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=287
[junit4] 2> 806100 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001 of type NRT
[junit4] 2> 806104 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 806105 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 806105 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 806106 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 806106 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6e6b8bac{/_m/xy,null,AVAILABLE}
[junit4] 2> 806107 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@234f6dae{HTTP/1.1,[http/1.1]}{127.0.0.1:45324}
[junit4] 2> 806107 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server Started @806302ms
[junit4] 2> 806107 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:39808/hdfs__localhost_39808__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/_m/xy, hostPort=45324, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/cores}
[junit4] 2> 806107 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 806108 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-30T00:58:27.750Z
[junit4] 2> 806116 INFO (zkConnectionManagerCallback-1113-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 806121 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 806121 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/solr.xml
[junit4] 2> 806127 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 806127 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 806129 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 806468 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41780/solr
[junit4] 2> 806500 INFO (zkConnectionManagerCallback-1117-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 806506 INFO (zkConnectionManagerCallback-1119-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 806519 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 806526 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 806526 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45324__m%2Fxy
[junit4] 2> 806527 INFO (zkCallback-1092-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 806527 INFO (zkCallback-1110-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 806527 INFO (zkCallback-1099-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 806544 INFO (zkCallback-1118-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 806567 INFO (zkConnectionManagerCallback-1126-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 806569 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 806571 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 806571 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 806618 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 806660 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 806660 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 806665 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/cores
[junit4] 2> 806729 INFO (qtp731027699-5603) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:45324__m%252Fxy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 806736 INFO (OverseerCollectionConfigSetProcessor-72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 806745 INFO (qtp731027699-5605) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 806748 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 806754 INFO (qtp2137760111-5658) [n:127.0.0.1:45324__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 806762 INFO (OverseerThreadFactory-2099-thread-3-processing-n:127.0.0.1:37095__m%2Fxy) [n:127.0.0.1:37095__m%2Fxy c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:45324__m%2Fxy for creating new replica of shard shard1 for collection collection1
[junit4] 2> 806784 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy x:collection1_shard1_replica_n21] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 807845 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 807880 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=test
[junit4] 2> 808055 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 808111 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
[junit4] 2> 808114 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 808114 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-1-001/cores/collection1_shard1_replica_n21/data/]
[junit4] 2> 808142 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=21.619140625, floorSegmentMB=0.26953125, forceMergeDeletesPctAllowed=22.12888844486297, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6395981875322819, deletesPctAllowed=45.80989708961823
[junit4] 2> 808155 WARN (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 808331 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 808331 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 808333 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 808333 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 808337 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=88.462890625, floorSegmentMB=1.7255859375, forceMergeDeletesPctAllowed=24.202588505252482, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6912498608789626, deletesPctAllowed=31.804024835151772
[junit4] 2> 808338 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@6b9d1fc4[collection1_shard1_replica_n21] main]
[junit4] 2> 808346 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 808347 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 808351 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 808351 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615709979272019968
[junit4] 2> 808361 INFO (searcherExecutor-2118-thread-1-processing-n:127.0.0.1:45324__m%2Fxy x:collection1_shard1_replica_n21 c:collection1 s:shard1) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@6b9d1fc4[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 808578 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node22=0}, version=0}
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45324/_m/xy/collection1_shard1_replica_n21/
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy http://127.0.0.1:45324/_m/xy/collection1_shard1_replica_n21/ has no replicas
[junit4] 2> 808582 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 808587 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45324/_m/xy/collection1_shard1_replica_n21/ shard1
[junit4] 2> 808739 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 808745 INFO (qtp2137760111-5653) [n:127.0.0.1:45324__m%2Fxy c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1961
[junit4] 2> 808749 INFO (qtp731027699-5603) [n:127.0.0.1:37095__m%2Fxy c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:45324__m%252Fxy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=2019
[junit4] 2> 809239 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-2-001 of type NRT
[junit4] 2> 809242 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 809301 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 809301 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 809301 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 809301 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@459cec24{/_m/xy,null,AVAILABLE}
[junit4] 2> 809302 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@381a485{HTTP/1.1,[http/1.1]}{127.0.0.1:40664}
[junit4] 2> 809302 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.e.j.s.Server Started @809497ms
[junit4] 2> 809302 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:39808/hdfs__localhost_39808__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001_tempDir-002_jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/_m/xy, hostPort=40664, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-2-001/cores}
[junit4] 2> 809303 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 809307 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-30T00:58:30.949Z
[junit4] 2> 809338 INFO (zkConnectionManagerCallback-1129-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 809340 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 809340 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-2-001/solr.xml
[junit4] 2> 809346 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 809346 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 809347 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 809727 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41780/solr
[junit4] 2> 809738 INFO (zkConnectionManagerCallback-1133-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 809744 INFO (zkConnectionManagerCallback-1135-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 809755 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 809764 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 809764 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40664__m%2Fxy
[junit4] 2> 809768 INFO (zkCallback-1092-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809768 INFO (zkCallback-1110-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809768 INFO (zkCallback-1099-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809768 INFO (zkCallback-1118-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809768 INFO (zkCallback-1125-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809795 INFO (zkCallback-1134-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 809818 INFO (zkConnectionManagerCallback-1142-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 809820 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 809823 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41780/solr ready
[junit4] 2> 809824 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 809860 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 809881 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 809881 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc893f5
[junit4] 2> 809884 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[949909B0C215D49]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_949909B0C215D49-001/shard-2-001/cores
[junit4] 2> 809941 INFO (qtp731027699-5604) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:40664__m%252Fxy&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 809952 INFO (OverseerCollectionConfigSetProcessor-72188396363120644-127.0.0.1:37095__m%2Fxy-n_0000000000) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 809968 INFO (qtp731027699-5603) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 809970 INFO (qtp731027699-5602) [n:127.0.0.1:37095__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 809985 INFO (qtp1985293797-5696) [n:127.0.0.1:40664__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 809988 INFO (qtp2137760111-5654) [n:127.0.0.1:45324__m%2Fxy ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.collection1.shard1.replica_n21:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 809993 INFO (qtp213776
[...truncated too long message...]
kout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1299618971
[copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1299618971
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1299618971
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 48ms :: artifacts dl 2ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 250 minutes 10 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-NightlyTests-7.x - Build # 361 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/361/
4 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test
Error Message:
There are still nodes recoverying - waited for 320 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
at __randomizedtesting.SeedInfo.seed([F900155FC4E8D440:71542A856A14B9B8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:920)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1477)
at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:145)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest: 1) Thread[id=73979, name=searcherExecutor-9510-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=73855, name=searcherExecutor-9468-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=74059, name=searcherExecutor-9530-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=73935, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=74094, name=searcherExecutor-9544-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=73898, name=searcherExecutor-9482-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=74129, name=searcherExecutor-9558-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest:
1) Thread[id=73979, name=searcherExecutor-9510-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=73855, name=searcherExecutor-9468-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=74059, name=searcherExecutor-9530-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=73935, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=74094, name=searcherExecutor-9544-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=73898, name=searcherExecutor-9482-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=74129, name=searcherExecutor-9558-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([F900155FC4E8D440]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=73979, name=searcherExecutor-9510-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=73855, name=searcherExecutor-9468-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=74059, name=searcherExecutor-9530-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=73935, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=74094, name=searcherExecutor-9544-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=73898, name=searcherExecutor-9482-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=74129, name=searcherExecutor-9558-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=73979, name=searcherExecutor-9510-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=73855, name=searcherExecutor-9468-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=74059, name=searcherExecutor-9530-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=73935, name=searcherExecutor-9496-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=74094, name=searcherExecutor-9544-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=73898, name=searcherExecutor-9482-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=74129, name=searcherExecutor-9558-thread-1, state=WAITING, group=TGRP-HdfsRestartWhileUpdatingTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([F900155FC4E8D440]:0)
FAILED: org.apache.solr.client.solrj.impl.CloudSolrClientTest.testRouting
Error Message:
Error from server at https://127.0.0.1:44030/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: <pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/> </body> </html>
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:44030/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
<pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
</body>
</html>
at __randomizedtesting.SeedInfo.seed([1D4E01942286A0E8:DFF93DFC21C65090]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testRouting(CloudSolrClientTest.java:238)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:44030/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
<pre> Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
</body>
</html>
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Build Log:
[...truncated 15600 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest
[junit4] 2> 3587569 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/init-core-data-001
[junit4] 2> 3587570 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3587573 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3587573 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3587612 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3587618 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3587620 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3587655 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_36929_hdfs____gftciv/webapp
[junit4] 2> 3588066 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36929
[junit4] 2> 3588154 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3588155 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3588177 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_33140_datanode____ry3w50/webapp
[junit4] 2> 3588593 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33140
[junit4] 2> 3588651 WARN (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3588652 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3588683 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_40985_datanode____4bf7xd/webapp
[junit4] 2> 3588728 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/tempDir-001/hdfsBaseDir/data/data2/]] heartbeating to localhost/127.0.0.1:37804) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 3588740 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6cf25facef291: from storage DS-ebed7a1a-1f24-4caf-a0c9-5368e57332ce node DatanodeRegistration(127.0.0.1:41525, datanodeUuid=d95058ce-0cf7-481e-a2de-76934e48249e, infoPort=42099, infoSecurePort=0, ipcPort=39795, storageInfo=lv=-56;cid=testClusterID;nsid=1850678792;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3588741 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6cf25facef291: from storage DS-93aab6fb-2a44-4ee3-b7d0-4874459362a8 node DatanodeRegistration(127.0.0.1:41525, datanodeUuid=d95058ce-0cf7-481e-a2de-76934e48249e, infoPort=42099, infoSecurePort=0, ipcPort=39795, storageInfo=lv=-56;cid=testClusterID;nsid=1850678792;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3589130 INFO (SUITE-HdfsRestartWhileUpdatingTest-seed#[F900155FC4E8D440]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40985
[junit4] 2> 3589258 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/tempDir-001/hdfsBaseDir/data/data4/]] heartbeating to localhost/127.0.0.1:37804) [ ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
[junit4] 2> 3589271 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6cf261a745818: from storage DS-d042ba87-bfe4-4b03-b952-06e6806a80bb node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=2966e4fd-6e7b-460a-8f46-1c3cd2d8bd22, infoPort=41526, infoSecurePort=0, ipcPort=33525, storageInfo=lv=-56;cid=testClusterID;nsid=1850678792;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3589272 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0x6cf261a745818: from storage DS-e07e0b27-5421-4d40-89a3-d8e8666ab47c node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=2966e4fd-6e7b-460a-8f46-1c3cd2d8bd22, infoPort=41526, infoSecurePort=0, ipcPort=33525, storageInfo=lv=-56;cid=testClusterID;nsid=1850678792;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3589457 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3589457 INFO (Thread-55028) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3589457 INFO (Thread-55028) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3589461 ERROR (Thread-55028) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3589557 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkTestServer start zk server on port:35789
[junit4] 2> 3589562 INFO (zkConnectionManagerCallback-6720-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589566 INFO (zkConnectionManagerCallback-6722-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589573 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3589575 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 3589577 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3589578 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3589580 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3589582 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3589583 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3589585 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3589586 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3589588 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3589589 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3589591 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 3589765 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3589769 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3589769 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3589769 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3589769 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2c07bd5a{/,null,AVAILABLE}
[junit4] 2> 3589770 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2cbbf67d{HTTP/1.1,[http/1.1]}{127.0.0.1:44864}
[junit4] 2> 3589770 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server Started @3589829ms
[junit4] 2> 3589770 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:37804/hdfs__localhost_37804__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=44864, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/cores}
[junit4] 2> 3589773 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3589773 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-29T02:57:07.150Z
[junit4] 2> 3589775 INFO (zkConnectionManagerCallback-6724-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589776 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3589777 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/solr.xml
[junit4] 2> 3589781 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3589781 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3589783 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3589963 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35789/solr
[junit4] 2> 3589964 INFO (zkConnectionManagerCallback-6728-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3589969 INFO (zkConnectionManagerCallback-6730-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3590082 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44864_
[junit4] 2> 3590084 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.Overseer Overseer (id=72183201161740292-127.0.0.1:44864_-n_0000000000) starting
[junit4] 2> 3590093 INFO (zkConnectionManagerCallback-6737-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3590095 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35789/solr ready
[junit4] 2> 3590096 INFO (OverseerStateUpdate-72183201161740292-127.0.0.1:44864_-n_0000000000) [n:127.0.0.1:44864_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:44864_
[junit4] 2> 3590104 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44864_
[junit4] 2> 3590112 INFO (zkCallback-6736-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3590112 INFO (zkCallback-6729-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3590134 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3590170 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3590188 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3590189 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3590191 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [n:127.0.0.1:44864_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/cores
[junit4] 2> 3590221 INFO (zkConnectionManagerCallback-6743-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3590223 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3590225 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35789/solr ready
[junit4] 2> 3590226 INFO (qtp379481793-73815) [n:127.0.0.1:44864_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44864_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3590234 INFO (OverseerThreadFactory-9463-thread-1-processing-n:127.0.0.1:44864_) [n:127.0.0.1:44864_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 3590343 INFO (qtp379481793-73810) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3590349 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ x:control_collection_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3590349 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ x:control_collection_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3591375 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 3591392 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 3591550 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 3591573 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 3591574 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3591574 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 3591583 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=1.73828125, floorSegmentMB=1.2861328125, forceMergeDeletesPctAllowed=17.251147289491627, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19017649322459562, deletesPctAllowed=36.714809707763436
[junit4] 2> 3591595 WARN (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3591687 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3591687 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3591693 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 3591693 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 3591695 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=0.7802734375, floorSegmentMB=0.8623046875, forceMergeDeletesPctAllowed=28.347754521320784, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.48483855280643395, deletesPctAllowed=25.385944100098243
[junit4] 2> 3591696 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1cd47ed[control_collection_shard1_replica_n1] main]
[junit4] 2> 3591702 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3591703 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3591704 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3591706 INFO (searcherExecutor-9468-thread-1-processing-n:127.0.0.1:44864_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@1cd47ed[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3591706 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615626847212535808
[junit4] 2> 3591716 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 3591722 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3591722 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3591722 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:44864/control_collection_shard1_replica_n1/
[junit4] 2> 3591722 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3591723 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:44864/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 3591723 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3591728 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:44864/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 3591832 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3591834 INFO (qtp379481793-73812) [n:127.0.0.1:44864_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1485
[junit4] 2> 3591840 INFO (qtp379481793-73815) [n:127.0.0.1:44864_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3591935 INFO (zkCallback-6729-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3592239 INFO (OverseerCollectionConfigSetProcessor-72183201161740292-127.0.0.1:44864_-n_0000000000) [n:127.0.0.1:44864_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3592840 INFO (qtp379481793-73815) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44864_&wt=javabin&version=2} status=0 QTime=2613
[junit4] 2> 3592845 INFO (zkConnectionManagerCallback-6748-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3592846 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3592848 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35789/solr ready
[junit4] 2> 3592848 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3592849 INFO (qtp379481793-73814) [n:127.0.0.1:44864_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3592852 INFO (OverseerThreadFactory-9463-thread-2-processing-n:127.0.0.1:44864_) [n:127.0.0.1:44864_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3593055 WARN (OverseerThreadFactory-9463-thread-2-processing-n:127.0.0.1:44864_) [n:127.0.0.1:44864_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 3593058 INFO (qtp379481793-73814) [n:127.0.0.1:44864_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3593058 INFO (qtp379481793-73814) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=209
[junit4] 2> 3593237 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001 of type NRT
[junit4] 2> 3593238 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3593239 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3593239 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3593239 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3593240 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@84f52b{/,null,AVAILABLE}
[junit4] 2> 3593240 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@23395dc9{HTTP/1.1,[http/1.1]}{127.0.0.1:36388}
[junit4] 2> 3593242 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server Started @3593301ms
[junit4] 2> 3593242 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:37804/hdfs__localhost_37804__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=36388, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/cores}
[junit4] 2> 3593244 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3593244 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-29T02:57:10.621Z
[junit4] 2> 3593246 INFO (zkConnectionManagerCallback-6750-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593247 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3593247 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/solr.xml
[junit4] 2> 3593252 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3593252 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3593299 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3593457 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35789/solr
[junit4] 2> 3593458 INFO (zkConnectionManagerCallback-6754-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593461 INFO (zkConnectionManagerCallback-6756-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593469 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3593474 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3593474 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36388_
[junit4] 2> 3593476 INFO (zkCallback-6736-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3593483 INFO (zkCallback-6729-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3593483 INFO (zkCallback-6747-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3593488 INFO (zkCallback-6755-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3593506 INFO (zkConnectionManagerCallback-6763-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3593507 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3593508 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35789/solr ready
[junit4] 2> 3593509 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3593539 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3593558 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3593559 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3593561 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/cores
[junit4] 2> 3593605 INFO (qtp36025874-73870) [n:127.0.0.1:36388_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:36388_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3593608 INFO (OverseerCollectionConfigSetProcessor-72183201161740292-127.0.0.1:44864_-n_0000000000) [n:127.0.0.1:44864_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3593611 INFO (qtp36025874-73865) [n:127.0.0.1:36388_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3593616 INFO (qtp379481793-73817) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=1
[junit4] 2> 3593618 INFO (qtp379481793-73817) [n:127.0.0.1:44864_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 3593618 INFO (OverseerThreadFactory-9463-thread-3-processing-n:127.0.0.1:44864_) [n:127.0.0.1:44864_ c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:36388_ for creating new replica of shard shard1 for collection collection1
[junit4] 2> 3593625 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ x:collection1_shard1_replica_n21] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3594649 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.6.0
[junit4] 2> 3594672 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=test
[junit4] 2> 3594791 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 3594810 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
[junit4] 2> 3594811 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@53cdc150
[junit4] 2> 3594811 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-1-001/cores/collection1_shard1_replica_n21/data/]
[junit4] 2> 3594816 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=1.73828125, floorSegmentMB=1.2861328125, forceMergeDeletesPctAllowed=17.251147289491627, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19017649322459562, deletesPctAllowed=36.714809707763436
[junit4] 2> 3594824 WARN (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3594895 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3594896 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3594897 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 30000ms;
[junit4] 2> 3594897 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: if uncommitted for 3000ms;
[junit4] 2> 3594900 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=0.7802734375, floorSegmentMB=0.8623046875, forceMergeDeletesPctAllowed=28.347754521320784, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.48483855280643395, deletesPctAllowed=25.385944100098243
[junit4] 2> 3594900 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@44694623[collection1_shard1_replica_n21] main]
[junit4] 2> 3594902 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3594902 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3594903 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3594903 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1615626850564833280
[junit4] 2> 3594907 INFO (searcherExecutor-9482-thread-1-processing-n:127.0.0.1:36388_ x:collection1_shard1_replica_n21 c:collection1 s:shard1) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@44694623[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3594911 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkShardTerms Successful update of terms at /collections/collection1/terms/shard1 to Terms{values={core_node22=0}, version=0}
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36388/collection1_shard1_replica_n21/
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy http://127.0.0.1:36388/collection1_shard1_replica_n21/ has no replicas
[junit4] 2> 3594915 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3594919 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36388/collection1_shard1_replica_n21/ shard1
[junit4] 2> 3595071 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3595074 INFO (qtp36025874-73866) [n:127.0.0.1:36388_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1449
[junit4] 2> 3595077 INFO (qtp36025874-73870) [n:127.0.0.1:36388_ c:collection1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:36388_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1471
[junit4] 2> 3595286 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-2-001 of type NRT
[junit4] 2> 3595288 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 3595289 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3595289 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3595289 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3595290 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44694501{/,null,AVAILABLE}
[junit4] 2> 3595290 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6177fc40{HTTP/1.1,[http/1.1]}{127.0.0.1:46131}
[junit4] 2> 3595290 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.e.j.s.Server Started @3595349ms
[junit4] 2> 3595290 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:37804/hdfs__localhost_37804__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-7.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001_tempDir-002_jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=46131, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRestartWhileUpdatingTest_F900155FC4E8D440-001/shard-2-001/cores}
[junit4] 2> 3595291 ERROR (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3595291 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3595291 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.6.0
[junit4] 2> 3595291 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3595291 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3595294 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-10-29T02:57:12.671Z
[junit4] 2> 3595297 INFO (zkConnectionManagerCallback-6766-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3595298 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3595298 INFO (TEST-HdfsRestartWhileUpdatingTest.test-seed#[F900155FC4E8D440]) [
[...truncated too long message...]
kout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1375865702
[copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1375865702
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/checkout/solr/null1375865702
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 39ms :: artifacts dl 2ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 290 minutes 33 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)