You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/08/21 02:21:55 UTC
[JENKINS-EA] Lucene-Solr-7.x-Linux (32bit/jdk-9-ea+181) - Build #
295 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/295/
Java: 32bit/jdk-9-ea+181 -client -XX:+UseParallelGC --illegal-access=deny
1 tests failed.
FAILED: org.apache.solr.cloud.HttpPartitionTest.test
Error Message:
Doc with id=1 not found in http://127.0.0.1:35191/collMinRf_1x3 due to: Path not found: /id; rsp={doc=null}
Stack Trace:
java.lang.AssertionError: Doc with id=1 not found in http://127.0.0.1:35191/collMinRf_1x3 due to: Path not found: /id; rsp={doc=null}
at __randomizedtesting.SeedInfo.seed([8E3A470CE083B308:66E78D64E7FDEF0]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:603)
at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:558)
at org.apache.solr.cloud.HttpPartitionTest.testMinRf(HttpPartitionTest.java:249)
at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:127)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Build Log:
[...truncated 11186 lines...]
[junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
[junit4] 2> 398208 INFO (SUITE-HttpPartitionTest-seed#[8E3A470CE083B308]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/init-core-data-001
[junit4] 2> 398208 WARN (SUITE-HttpPartitionTest-seed#[8E3A470CE083B308]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 398208 INFO (SUITE-HttpPartitionTest-seed#[8E3A470CE083B308]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 398209 INFO (SUITE-HttpPartitionTest-seed#[8E3A470CE083B308]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl="https://issues.apache.org/jira/browse/SOLR-5776")
[junit4] 2> 398209 INFO (SUITE-HttpPartitionTest-seed#[8E3A470CE083B308]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 398210 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 398210 INFO (Thread-865) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 398210 INFO (Thread-865) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 398213 ERROR (Thread-865) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 398310 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkTestServer start zk server on port:36865
[junit4] 2> 398316 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 398317 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 398317 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 398318 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 398318 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 398318 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 398319 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 398319 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 398320 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 398320 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 398320 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 398321 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 398410 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 398411 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@95fdec{/,null,AVAILABLE}
[junit4] 2> 398412 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@18c0af2{HTTP/1.1,[http/1.1]}{127.0.0.1:37141}
[junit4] 2> 398412 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.Server Started @399655ms
[junit4] 2> 398412 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/tempDir-001/control/data, hostPort=33783, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/control-001/cores, replicaType=NRT}
[junit4] 2> 398412 ERROR (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 398412 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 398412 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 398412 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 398412 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-21T01:38:13.136759Z
[junit4] 2> 398415 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 398415 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/control-001/solr.xml
[junit4] 2> 398417 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 398419 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36865/solr
[junit4] 2> 398452 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 398452 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33783_
[junit4] 2> 398452 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.Overseer Overseer (id=98518924848988164-127.0.0.1:33783_-n_0000000000) starting
[junit4] 2> 398460 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33783_
[junit4] 2> 398461 INFO (zkCallback-456-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 398510 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 398517 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 398517 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 398518 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/control-001/cores
[junit4] 2> 398530 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 398532 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36865/solr ready
[junit4] 2> 398536 INFO (SocketProxy-Acceptor-33783) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=48576,localport=33783], receiveBufferSize:531000
[junit4] 2> 398538 INFO (SocketProxy-Acceptor-33783) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37141,localport=47506], receiveBufferSize=530904
[junit4] 2> 398539 INFO (qtp23591168-3329) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33783_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 398544 INFO (OverseerThreadFactory-1709-thread-1) [ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 398647 INFO (SocketProxy-Acceptor-33783) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=48580,localport=33783], receiveBufferSize:531000
[junit4] 2> 398647 INFO (SocketProxy-Acceptor-33783) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37141,localport=47510], receiveBufferSize=530904
[junit4] 2> 398648 INFO (qtp23591168-3331) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 398649 INFO (qtp23591168-3331) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 398752 INFO (zkCallback-456-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 399658 INFO (qtp23591168-3331) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 399667 INFO (qtp23591168-3331) [ ] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 399740 INFO (qtp23591168-3331) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 399746 INFO (qtp23591168-3331) [ ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 399747 INFO (qtp23591168-3331) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 399747 INFO (qtp23591168-3331) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 399747 INFO (qtp23591168-3331) [ ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 399749 INFO (qtp23591168-3331) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=71.5263671875, floorSegmentMB=1.2080078125, forceMergeDeletesPctAllowed=16.257453184736036, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6962623759478992
[junit4] 2> 399750 WARN (qtp23591168-3331) [ ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 399780 INFO (qtp23591168-3331) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 399780 INFO (qtp23591168-3331) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 399781 INFO (qtp23591168-3331) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 399781 INFO (qtp23591168-3331) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 399782 INFO (qtp23591168-3331) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=68.802734375, floorSegmentMB=0.291015625, forceMergeDeletesPctAllowed=20.051320193750794, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 399782 INFO (qtp23591168-3331) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@99bdc1[control_collection_shard1_replica_n1] main]
[junit4] 2> 399785 INFO (qtp23591168-3331) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 399785 INFO (qtp23591168-3331) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 399785 INFO (qtp23591168-3331) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 399786 INFO (searcherExecutor-1712-thread-1) [ ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@99bdc1[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 399786 INFO (qtp23591168-3331) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576302799235317760
[junit4] 2> 399797 INFO (qtp23591168-3331) [ ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 399797 INFO (qtp23591168-3331) [ ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 399797 INFO (qtp23591168-3331) [ ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:33783/control_collection_shard1_replica_n1/
[junit4] 2> 399797 INFO (qtp23591168-3331) [ ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 399797 INFO (qtp23591168-3331) [ ] o.a.s.c.SyncStrategy http://127.0.0.1:33783/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 399797 INFO (qtp23591168-3331) [ ] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 399799 INFO (qtp23591168-3331) [ ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:33783/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 399900 INFO (zkCallback-456-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 399949 INFO (qtp23591168-3331) [ ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 399950 INFO (qtp23591168-3331) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1302
[junit4] 2> 399952 INFO (qtp23591168-3329) [ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 400051 INFO (zkCallback-456-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 400546 INFO (OverseerCollectionConfigSetProcessor-98518924848988164-127.0.0.1:33783_-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 400952 INFO (qtp23591168-3329) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:33783_&wt=javabin&version=2} status=0 QTime=2412
[junit4] 2> 400962 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 400962 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:36865/solr ready
[junit4] 2> 400962 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 400963 INFO (SocketProxy-Acceptor-33783) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=48608,localport=33783], receiveBufferSize:531000
[junit4] 2> 400963 INFO (SocketProxy-Acceptor-33783) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37141,localport=47538], receiveBufferSize=530904
[junit4] 2> 400963 INFO (qtp23591168-3329) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 400964 INFO (OverseerThreadFactory-1709-thread-2) [ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 400964 WARN (OverseerThreadFactory-1709-thread-2) [ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 401166 INFO (qtp23591168-3329) [ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 401166 INFO (qtp23591168-3329) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=203
[junit4] 2> 401249 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-1-001 of type TLOG
[junit4] 2> 401250 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 401251 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d2a6b4{/,null,AVAILABLE}
[junit4] 2> 401252 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@16e5444{HTTP/1.1,[http/1.1]}{127.0.0.1:40999}
[junit4] 2> 401252 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.Server Started @402495ms
[junit4] 2> 401252 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/tempDir-001/jetty1, hostPort=35191, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-1-001/cores, replicaType=TLOG}
[junit4] 2> 401252 ERROR (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 401252 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 401252 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 401252 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 401252 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-21T01:38:15.976751Z
[junit4] 2> 401254 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 401254 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-1-001/solr.xml
[junit4] 2> 401256 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 401258 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36865/solr
[junit4] 2> 401262 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 401263 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 401263 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35191_
[junit4] 2> 401264 INFO (zkCallback-456-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 401264 INFO (zkCallback-463-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 401264 INFO (zkCallback-468-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 401338 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 401346 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 401346 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 401347 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-1-001/cores
[junit4] 2> 401364 INFO (qtp23591168-3330) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35191_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 401368 INFO (OverseerThreadFactory-1709-thread-3) [ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:35191_ for creating new replica
[junit4] 2> 401368 INFO (OverseerCollectionConfigSetProcessor-98518924848988164-127.0.0.1:33783_-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 401373 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=47714,localport=35191], receiveBufferSize:531000
[junit4] 2> 401374 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=40999,localport=50988], receiveBufferSize=530904
[junit4] 2> 401377 INFO (qtp9551482-3377) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 401377 INFO (qtp9551482-3377) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 402391 INFO (qtp9551482-3377) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 402421 INFO (qtp9551482-3377) [ ] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 402482 INFO (qtp9551482-3377) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 402491 INFO (qtp9551482-3377) [ ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 402491 INFO (qtp9551482-3377) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 402491 INFO (qtp9551482-3377) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 402491 INFO (qtp9551482-3377) [ ] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 402493 INFO (qtp9551482-3377) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=71.5263671875, floorSegmentMB=1.2080078125, forceMergeDeletesPctAllowed=16.257453184736036, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6962623759478992
[junit4] 2> 402494 WARN (qtp9551482-3377) [ ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 402525 INFO (qtp9551482-3377) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 402525 INFO (qtp9551482-3377) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 402526 INFO (qtp9551482-3377) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 402526 INFO (qtp9551482-3377) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 402527 INFO (qtp9551482-3377) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=68.802734375, floorSegmentMB=0.291015625, forceMergeDeletesPctAllowed=20.051320193750794, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 402527 INFO (qtp9551482-3377) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1bf52[collection1_shard2_replica_t41] main]
[junit4] 2> 402528 INFO (qtp9551482-3377) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 402528 INFO (qtp9551482-3377) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 402529 INFO (qtp9551482-3377) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 402529 INFO (searcherExecutor-1723-thread-1) [ ] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@1bf52[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 402529 INFO (qtp9551482-3377) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576302802111561728
[junit4] 2> 402532 INFO (qtp9551482-3377) [ ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 402532 INFO (qtp9551482-3377) [ ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 402532 INFO (qtp9551482-3377) [ ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35191/collection1_shard2_replica_t41/
[junit4] 2> 402533 INFO (qtp9551482-3377) [ ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 402533 INFO (qtp9551482-3377) [ ] o.a.s.c.SyncStrategy http://127.0.0.1:35191/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 402533 INFO (qtp9551482-3377) [ ] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 402533 INFO (qtp9551482-3377) [ ] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 402534 INFO (qtp9551482-3377) [ ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35191/collection1_shard2_replica_t41/ shard2
[junit4] 2> 402684 INFO (qtp9551482-3377) [ ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 402685 INFO (qtp9551482-3377) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1308
[junit4] 2> 402700 INFO (qtp23591168-3330) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35191_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1335
[junit4] 2> 402794 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-2-001 of type TLOG
[junit4] 2> 402795 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 402795 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1c000f3{/,null,AVAILABLE}
[junit4] 2> 402796 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@14ce8b1{HTTP/1.1,[http/1.1]}{127.0.0.1:37541}
[junit4] 2> 402796 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.Server Started @404039ms
[junit4] 2> 402796 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/tempDir-001/jetty2, hostPort=44517, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-2-001/cores, replicaType=TLOG}
[junit4] 2> 402796 ERROR (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 402796 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 402796 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 402796 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 402796 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-21T01:38:17.520895Z
[junit4] 2> 402798 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 402798 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-2-001/solr.xml
[junit4] 2> 402801 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 402804 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36865/solr
[junit4] 2> 402809 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 402810 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 402811 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44517_
[junit4] 2> 402811 INFO (zkCallback-463-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 402811 INFO (zkCallback-468-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 402812 INFO (zkCallback-456-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 402812 INFO (zkCallback-474-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 402882 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 402895 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 402895 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 402897 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-2-001/cores
[junit4] 2> 402936 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36856,localport=44517], receiveBufferSize:531000
[junit4] 2> 402941 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37541,localport=45716], receiveBufferSize=530904
[junit4] 2> 402944 INFO (qtp22807782-3410) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:44517_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 402948 INFO (OverseerThreadFactory-1709-thread-4) [ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:44517_ for creating new replica
[junit4] 2> 402948 INFO (OverseerCollectionConfigSetProcessor-98518924848988164-127.0.0.1:33783_-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 402949 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36860,localport=44517], receiveBufferSize:531000
[junit4] 2> 402950 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37541,localport=45720], receiveBufferSize=530904
[junit4] 2> 402950 INFO (qtp22807782-3406) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 402951 INFO (qtp22807782-3406) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 403958 INFO (qtp22807782-3406) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 403965 INFO (qtp22807782-3406) [ ] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 404014 INFO (qtp22807782-3406) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 404020 INFO (qtp22807782-3406) [ ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 404021 INFO (qtp22807782-3406) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 404021 INFO (qtp22807782-3406) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 404021 INFO (qtp22807782-3406) [ ] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 404023 INFO (qtp22807782-3406) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=71.5263671875, floorSegmentMB=1.2080078125, forceMergeDeletesPctAllowed=16.257453184736036, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6962623759478992
[junit4] 2> 404024 WARN (qtp22807782-3406) [ ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 404054 INFO (qtp22807782-3406) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 404054 INFO (qtp22807782-3406) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 404055 INFO (qtp22807782-3406) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 404055 INFO (qtp22807782-3406) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 404056 INFO (qtp22807782-3406) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=68.802734375, floorSegmentMB=0.291015625, forceMergeDeletesPctAllowed=20.051320193750794, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 404056 INFO (qtp22807782-3406) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@16d4c08[collection1_shard1_replica_t43] main]
[junit4] 2> 404057 INFO (qtp22807782-3406) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 404057 INFO (qtp22807782-3406) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 404058 INFO (qtp22807782-3406) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 404059 INFO (searcherExecutor-1734-thread-1) [ ] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@16d4c08[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 404059 INFO (qtp22807782-3406) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576302803715883008
[junit4] 2> 404062 INFO (qtp22807782-3406) [ ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 404062 INFO (qtp22807782-3406) [ ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 404062 INFO (qtp22807782-3406) [ ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:44517/collection1_shard1_replica_t43/
[junit4] 2> 404062 INFO (qtp22807782-3406) [ ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 404062 INFO (qtp22807782-3406) [ ] o.a.s.c.SyncStrategy http://127.0.0.1:44517/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 404062 INFO (qtp22807782-3406) [ ] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 404062 INFO (qtp22807782-3406) [ ] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 404063 INFO (qtp22807782-3406) [ ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:44517/collection1_shard1_replica_t43/ shard1
[junit4] 2> 404213 INFO (qtp22807782-3406) [ ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 404220 INFO (qtp22807782-3406) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1270
[junit4] 2> 404225 INFO (qtp22807782-3410) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:44517_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1280
[junit4] 2> 404376 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-3-001 of type TLOG
[junit4] 2> 404377 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 404394 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6974e3{/,null,AVAILABLE}
[junit4] 2> 404395 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@135e851{HTTP/1.1,[http/1.1]}{127.0.0.1:43193}
[junit4] 2> 404395 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.Server Started @405638ms
[junit4] 2> 404395 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/tempDir-001/jetty3, hostPort=39725, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-3-001/cores, replicaType=TLOG}
[junit4] 2> 404395 ERROR (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 404395 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 404395 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 404395 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 404395 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-21T01:38:19.119829Z
[junit4] 2> 404397 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 404397 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-3-001/solr.xml
[junit4] 2> 404400 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 404405 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36865/solr
[junit4] 2> 404410 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 404411 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 404411 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39725_
[junit4] 2> 404412 INFO (zkCallback-474-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 404412 INFO (zkCallback-468-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 404412 INFO (zkCallback-463-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 404412 INFO (zkCallback-456-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 404413 INFO (zkCallback-480-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 404666 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 404678 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 404678 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 404680 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-3-001/cores
[junit4] 2> 404762 INFO (qtp22807782-3411) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39725_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 404766 INFO (OverseerCollectionConfigSetProcessor-98518924848988164-127.0.0.1:33783_-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 404766 INFO (OverseerThreadFactory-1709-thread-5) [ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:39725_ for creating new replica
[junit4] 2> 404768 INFO (SocketProxy-Acceptor-39725) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=34746,localport=39725], receiveBufferSize:531000
[junit4] 2> 404769 INFO (SocketProxy-Acceptor-39725) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43193,localport=37486], receiveBufferSize=530904
[junit4] 2> 404770 INFO (qtp3994980-3446) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 404770 INFO (qtp3994980-3446) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 405784 INFO (qtp3994980-3446) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 405816 INFO (qtp3994980-3446) [ ] o.a.s.s.IndexSchema [collection1_shard2_replica_t45] Schema name=test
[junit4] 2> 405882 INFO (qtp3994980-3446) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 405898 INFO (qtp3994980-3446) [ ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t45' using configuration from collection collection1, trusted=true
[junit4] 2> 405899 INFO (qtp3994980-3446) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t45' (registry 'solr.core.collection1.shard2.replica_t45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 405899 INFO (qtp3994980-3446) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 405899 INFO (qtp3994980-3446) [ ] o.a.s.c.SolrCore [[collection1_shard2_replica_t45] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-3-001/cores/collection1_shard2_replica_t45], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-3-001/cores/collection1_shard2_replica_t45/data/]
[junit4] 2> 405902 INFO (qtp3994980-3446) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=71.5263671875, floorSegmentMB=1.2080078125, forceMergeDeletesPctAllowed=16.257453184736036, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6962623759478992
[junit4] 2> 405905 WARN (qtp3994980-3446) [ ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 405952 INFO (qtp3994980-3446) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 405952 INFO (qtp3994980-3446) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 405954 INFO (qtp3994980-3446) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 405954 INFO (qtp3994980-3446) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 405955 INFO (qtp3994980-3446) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=68.802734375, floorSegmentMB=0.291015625, forceMergeDeletesPctAllowed=20.051320193750794, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 405955 INFO (qtp3994980-3446) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2cafb9[collection1_shard2_replica_t45] main]
[junit4] 2> 405956 INFO (qtp3994980-3446) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 405957 INFO (qtp3994980-3446) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 405957 INFO (qtp3994980-3446) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 405958 INFO (searcherExecutor-1745-thread-1) [ ] o.a.s.c.SolrCore [collection1_shard2_replica_t45] Registered new searcher Searcher@2cafb9[collection1_shard2_replica_t45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 405958 INFO (qtp3994980-3446) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576302805707128832
[junit4] 2> 405962 INFO (qtp3994980-3446) [ ] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_t45
[junit4] 2> 405963 INFO (updateExecutor-477-thread-1) [ ] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 405964 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 405964 INFO (qtp3994980-3446) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1193
[junit4] 2> 405965 INFO (qtp22807782-3411) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:39725_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1202
[junit4] 2> 405965 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 405965 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.ZkController collection1_shard2_replica_t45 stopping background replication from leader
[junit4] 2> 405965 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_t45]
[junit4] 2> 405966 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 405966 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_t45] as recovering, leader is [http://127.0.0.1:35191/collection1_shard2_replica_t41/] and I am [http://127.0.0.1:39725/collection1_shard2_replica_t45/]
[junit4] 2> 405970 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 405970 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:35191]; [WaitForState: action=PREPRECOVERY&core=collection1_shard2_replica_t41&nodeName=127.0.0.1:39725_&coreNodeName=core_node46&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 405970 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Wait for recoveries to finish - wait 30000 for each attempt
[junit4] 2> 405970 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30000
[junit4] 2> 405970 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=47802,localport=35191], receiveBufferSize:531000
[junit4] 2> 405973 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=40999,localport=51076], receiveBufferSize=530904
[junit4] 2> 405973 INFO (qtp9551482-3379) [ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node46, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 405973 INFO (qtp9551482-3379) [ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_t41, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:39725_, coreNodeName=core_node46, onlyIfActiveCheckResult=false, nodeProps: core_node46:{"core":"collection1_shard2_replica_t45","base_url":"http://127.0.0.1:39725","node_name":"127.0.0.1:39725_","state":"down","type":"TLOG"}
[junit4] 2> 406767 INFO (OverseerCollectionConfigSetProcessor-98518924848988164-127.0.0.1:33783_-n_0000000000) [ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 406974 INFO (qtp9551482-3379) [ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_t41, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:39725_, coreNodeName=core_node46, onlyIfActiveCheckResult=false, nodeProps: core_node46:{"core":"collection1_shard2_replica_t45","base_url":"http://127.0.0.1:39725","node_name":"127.0.0.1:39725_","state":"recovering","type":"TLOG"}
[junit4] 2> 406974 INFO (qtp9551482-3379) [ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node46, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 406974 INFO (qtp9551482-3379) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:39725_&onlyIfLeaderActive=true&core=collection1_shard2_replica_t41&coreNodeName=core_node46&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1000
[junit4] 2> 407474 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
[junit4] 2> 407474 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Attempting to replicate from [http://127.0.0.1:35191/collection1_shard2_replica_t41/].
[junit4] 2> 407475 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=47814,localport=35191], receiveBufferSize:531000
[junit4] 2> 407476 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=40999,localport=51088], receiveBufferSize=530904
[junit4] 2> 407476 INFO (qtp9551482-3379) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576302807298867200,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 407476 INFO (qtp9551482-3379) [ ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 407477 INFO (qtp9551482-3379) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 407477 INFO (qtp9551482-3379) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t41] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2}{commit=} 0 0
[junit4] 2> 407478 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=47818,localport=35191], receiveBufferSize:531000
[junit4] 2> 407478 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=40999,localport=51092], receiveBufferSize=530904
[junit4] 2> 407480 INFO (qtp9551482-3375) [ ] o.a.s.c.S.Request [collection1_shard2_replica_t41] webapp= path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 407481 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 407481 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 407481 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 407481 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 407481 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Replication Recovery was successful.
[junit4] 2> 407481 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 407481 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.ZkController collection1_shard2_replica_t45 starting background replication from leader
[junit4] 2> 407481 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 407482 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 407483 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Updating version bucket highest from index after successful recovery.
[junit4] 2> 407483 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576302807306207232
[junit4] 2> 407484 INFO (recoveryExecutor-478-thread-1) [ ] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[true]
[junit4] 2> 407970 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: collection1
[junit4] 2> 407972 INFO (qtp23591168-3326) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576302807818960896,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 407972 INFO (qtp23591168-3326) [ ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 407973 INFO (qtp23591168-3326) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 407973 INFO (qtp23591168-3326) [ ] o.a.s.u.p.LogUpdateProcessorFactory [control_collection_shard1_replica_n1] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 0
[junit4] 2> 407973 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=47824,localport=35191], receiveBufferSize:531000
[junit4] 2> 407973 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=40999,localport=51098], receiveBufferSize=530904
[junit4] 2> 407975 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36956,localport=44517], receiveBufferSize:531000
[junit4] 2> 407975 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37541,localport=45816], receiveBufferSize=530904
[junit4] 2> 407978 INFO (qtp22807782-3406) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576302807825252352,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 407978 INFO (qtp22807782-3406) [ ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 407978 INFO (qtp22807782-3406) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 407979 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=47834,localport=35191], receiveBufferSize:531000
[junit4] 2> 407979 INFO (qtp22807782-3406) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard1_replica_t43] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:35191/collection1_shard2_replica_t41/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 2
[junit4] 2> 407980 INFO (SocketProxy-Acceptor-39725) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=34804,localport=39725], receiveBufferSize:531000
[junit4] 2> 407981 INFO (SocketProxy-Acceptor-39725) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43193,localport=37548], receiveBufferSize=530904
[junit4] 2> 407981 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=40999,localport=51108], receiveBufferSize=530904
[junit4] 2> 407982 INFO (qtp9551482-3372) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1576302807829446656,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 407983 INFO (qtp3994980-3441) [ ] o.a.s.u.TestInjection Start waiting for replica in sync with leader
[junit4] 2> 407985 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=47844,localport=35191], receiveBufferSize:531000
[junit4] 2> 407985 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=40999,localport=51118], receiveBufferSize=530904
[junit4] 2> 407983 INFO (qtp9551482-3372) [ ] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 408014 INFO (qtp9551482-3374) [ ] o.a.s.c.S.Request [collection1_shard2_replica_t41] webapp= path=/replication params={qt=/replication&wt=javabin&version=2&command=details} status=0 QTime=0
[junit4] 2> 408016 INFO (qtp9551482-3372) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 408016 INFO (qtp9551482-3372) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t41] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:35191/collection1_shard2_replica_t41/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 34
[junit4] 2> 408024 INFO (qtp3994980-3441) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t45] webapp= path=/update params={update.distrib=FROMLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:35191/collection1_shard2_replica_t41/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 42
[junit4] 2> 408025 INFO (qtp9551482-3373) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t41] webapp= path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 50
[junit4] 2> 408026 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=47850,localport=35191], receiveBufferSize:531000
[junit4] 2> 408032 INFO (SocketProxy-Acceptor-35191) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=40999,localport=51124], receiveBufferSize=530904
[junit4] 2> 408033 INFO (qtp9551482-3372) [ ] o.a.s.c.S.Request [collection1_shard2_replica_t41] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 408034 INFO (SocketProxy-Acceptor-39725) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=34826,localport=39725], receiveBufferSize:531000
[junit4] 2> 408034 INFO (SocketProxy-Acceptor-39725) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=43193,localport=37566], receiveBufferSize=530904
[junit4] 2> 408035 INFO (qtp3994980-3446) [ ] o.a.s.c.S.Request [collection1_shard2_replica_t45] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 408036 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36986,localport=44517], receiveBufferSize:531000
[junit4] 2> 408037 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37541,localport=45846], receiveBufferSize=530904
[junit4] 2> 408037 INFO (qtp22807782-3411) [ ] o.a.s.c.S.Request [collection1_shard1_replica_t43] webapp= path=/select params={q=*:*&distrib=false&tests=checkShardConsistency&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 409264 INFO (indexFetcher-1751-thread-1) [ ] o.a.s.h.IndexFetcher Updated masterUrl to http://127.0.0.1:35191/collection1_shard2_replica_t41/
[junit4] 2> 409265 INFO (qtp9551482-3373) [ ] o.a.s.c.S.Request [collection1_shard2_replica_t41] webapp= path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 409266 INFO (indexFetcher-1751-thread-1) [ ] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 409266 INFO (indexFetcher-1751-thread-1) [ ] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 409266 INFO (indexFetcher-1751-thread-1) [ ] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 409266 INFO (indexFetcher-1751-thread-1) [ ] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 410038 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection with stateFormat=1: c8n_crud_1x2
[junit4] 2> 410040 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=37006,localport=44517], receiveBufferSize:531000
[junit4] 2> 410041 INFO (SocketProxy-Acceptor-44517) [ ] o.a.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=37541,localport=45866], receiveBufferSize=530904
[junit4] 2> 410042 INFO (qtp22807782-3410) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&maxShardsPerNode=1&collection.configName=conf1&nrtReplicas=0&name=c8n_crud_1x2&action=CREATE&numShards=1&tlogReplicas=2&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 410044 INFO (OverseerThreadFactory-1709-thread-5) [ ] o.a.s.c.CreateCollectionCmd Create collection c8n_crud_1x2
[junit4] 2> 410150 INFO (qtp9551482-3378) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=c8n_crud_1x2_shard1_replica_t1&action=CREATE&numShards=1&collection=c8n_crud_1x2&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 410151 INFO (qtp23591168-3324) [ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=c8n_crud_1x2_shard1_replica_t2&action=CREATE&numShards=1&collection=c8n_crud_1x2&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 411160 INFO (qtp9551482-3378) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 411161 INFO (qtp23591168-3324) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 411181 INFO (qtp9551482-3378) [ ] o.a.s.s.IndexSchema [c8n_crud_1x2_shard1_replica_t1] Schema name=test
[junit4] 2> 411182 INFO (qtp23591168-3324) [ ] o.a.s.s.IndexSchema [c8n_crud_1x2_shard1_replica_t2] Schema name=test
[junit4] 2> 411238 INFO (qtp9551482-3378) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 411244 INFO (qtp23591168-3324) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 411245 INFO (qtp9551482-3378) [ ] o.a.s.c.CoreContainer Creating SolrCore 'c8n_crud_1x2_shard1_replica_t1' using configuration from collection c8n_crud_1x2, trusted=true
[junit4] 2> 411246 INFO (qtp9551482-3378) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.c8n_crud_1x2.shard1.replica_t1' (registry 'solr.core.c8n_crud_1x2.shard1.replica_t1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 411246 INFO (qtp9551482-3378) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 411246 INFO (qtp9551482-3378) [ ] o.a.s.c.SolrCore [[c8n_crud_1x2_shard1_replica_t1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-1-001/cores/c8n_crud_1x2_shard1_replica_t1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/shard-1-001/cores/c8n_crud_1x2_shard1_replica_t1/data/]
[junit4] 2> 411248 INFO (qtp9551482-3378) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=71.5263671875, floorSegmentMB=1.2080078125, forceMergeDeletesPctAllowed=16.257453184736036, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6962623759478992
[junit4] 2> 411258 WARN (qtp9551482-3378) [ ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 411260 INFO (qtp23591168-3324) [ ] o.a.s.c.CoreContainer Creating SolrCore 'c8n_crud_1x2_shard1_replica_t2' using configuration from collection c8n_crud_1x2, trusted=true
[junit4] 2> 411260 INFO (qtp23591168-3324) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.c8n_crud_1x2.shard1.replica_t2' (registry 'solr.core.c8n_crud_1x2.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@191f681
[junit4] 2> 411260 INFO (qtp23591168-3324) [ ] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 411260 INFO (qtp23591168-3324) [ ] o.a.s.c.SolrCore [[c8n_crud_1x2_shard1_replica_t2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/control-001/cores/c8n_crud_1x2_shard1_replica_t2], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001/control-001/cores/c8n_crud_1x2_shard1_replica_t2/data/]
[junit4] 2> 411263 INFO (qtp23591168-3324) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=71.5263671875, floorSegmentMB=1.2080078125, forceMergeDeletesPctAllowed=16.257453184736036, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6962623759478992
[junit4] 2> 411264 WARN (qtp23591168-3324) [ ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 411292 INFO (qtp9551482-3378) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 411292 INFO (qtp9551482-3378) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 411293 INFO (qtp9551482-3378) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 411293 INFO (qtp9551482-3378) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 411294 INFO (qtp9551482-3378) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=68.802734375, floorSegmentMB=0.291015625, forceMergeDeletesPctAllowed=20.051320193750794, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 411294 INFO (qtp9551482-3378) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1661d87[c8n_crud_1x2_shard1_replica_t1] main]
[junit4] 2> 411295 INFO (qtp9551482-3378) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 411295 INFO (qtp9551482-3378) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 411296 INFO (qtp9551482-3378) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 411296 INFO (qtp9551482-3378) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576302811304427520
[junit4] 2> 411297 INFO (searcherExecutor-1752-thread-1) [ ] o.a.s.c.SolrCore [c8n_crud_1x2_shard1_replica_t1] Registered new searcher Searcher@1661d87[c8n_crud_1x2_shard1_replica_t1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 411299 INFO (qtp9551482-3378) [ ] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 411302 INFO (qtp23591168-3324) [ ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 411302 INFO (qtp23591168-3324) [ ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 411303 INFO (qtp23591168-3324) [ ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 411303 INFO (qtp23591168-3324) [ ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 411304 INFO (qtp23591168-3324) [ ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=68.802734375, floorSegmentMB=0.291015625, forceMergeDeletesPctAllowed=20.051320193750794, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 411304 INFO (qtp23591168-3324) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@735c0d[c8n_crud_1x2_shard1_replica_t2] main]
[junit4] 2> 411305 INFO (qtp23591168-3324) [ ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 411305 INFO (qtp23591168-3324) [ ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 411306 INFO (qtp23591168-3324) [ ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 411306 INFO (qtp23591168-3324) [ ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576302811314913280
[junit4] 2> 411306 INFO (searcherExecutor-1753-thread-1) [ ] o.a.s.c.SolrCore [c8n_crud_1x2_shard1_replica_t2] Registered new searcher Searcher@735c0d[c8n_crud_1x2_shard1_replica_t2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 411800 INFO (qtp9551482-3378) [ ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 411800 INFO (qtp9551482-3378) [ ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 411800 INFO (qtp9551482-3378) [ ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35191/c8n_crud_1x2_shard1_replica_t1/
[junit4] 2> 411800 INFO (qtp9551482-3378) [ ] o.a.s.u.PeerSync PeerSync: core=c8n_crud_1x2_shard1_replica_t1 url=http://127.0.0.1:35191 START replicas=[http://127.0.0.1:33783/c8n_crud_1x2_shard1_replica_t2/] nUpdates=100
[junit4] 2> 411800 INFO (SocketProxy-Acceptor-33783) [ ] o.a.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=48808,localport=33783], receiveBufferSize:5310
[...truncated too long message...]
st.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 427558 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@d0b40b: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@f59c
[junit4] 2> 427562 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 427562 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@17646c5: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@152a932
[junit4] 2> 427565 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 427565 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@d6085a: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@1ed2f42
[junit4] 2> 427567 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 427568 INFO (coreCloseExecutor-1796-thread-1) [ ] o.a.s.c.SolrCore [collection1_shard2_replica_t45] CLOSING SolrCore org.apache.solr.core.SolrCore@fae7a9
[junit4] 2> 427568 INFO (coreCloseExecutor-1796-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard2.replica_t45, tag=16443305
[junit4] 2> 427568 INFO (coreCloseExecutor-1796-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1672d5b: rootName = null, domain = solr.core.collection1.shard2.replica_t45, service url = null, agent id = null] for registry solr.core.collection1.shard2.replica_t45 / com.codahale.metrics.MetricRegistry@2e4c28
[junit4] 2> 427576 INFO (coreCloseExecutor-1796-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard2.leader, tag=16443305
[junit4] 2> 427581 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.Overseer Overseer (id=98518924848988175-127.0.0.1:39725_-n_0000000003) closing
[junit4] 2> 427581 INFO (OverseerStateUpdate-98518924848988175-127.0.0.1:39725_-n_0000000003) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:39725_
[junit4] 2> 429085 WARN (zkCallback-480-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 429085 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@135e851{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 429085 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6974e3{/,null,UNAVAILABLE}
[junit4] 2> 429086 ERROR (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 429086 INFO (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36865 36865
[junit4] 2> 429149 INFO (Thread-865) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36865 36865
[junit4] 2> 429150 WARN (Thread-865) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf1
[junit4] 2> 3 /solr/collections/collection1/state.json
[junit4] 2> 3 /solr/collections/collMinRf_1x3/state.json
[junit4] 2> 2 /solr/collections/c8n_crud_1x2/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/live_nodes
[junit4] 2> 6 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> 429150 WARN (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SocketProxy Closing 2 connections to: http://127.0.0.1:44517/, target: http://127.0.0.1:37541/
[junit4] 2> 429150 WARN (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SocketProxy Closing 12 connections to: http://127.0.0.1:35191/, target: http://127.0.0.1:40999/
[junit4] 2> 429150 WARN (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SocketProxy Closing 3 connections to: http://127.0.0.1:39725/, target: http://127.0.0.1:43193/
[junit4] 2> 429150 WARN (TEST-HttpPartitionTest.test-seed#[8E3A470CE083B308]) [ ] o.a.s.c.SocketProxy Closing 9 connections to: http://127.0.0.1:33783/, target: http://127.0.0.1:37141/
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HttpPartitionTest -Dtests.method=test -Dtests.seed=8E3A470CE083B308 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en-ZW -Dtests.timezone=Europe/Belgrade -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 30.9s J1 | HttpPartitionTest.test <<<
[junit4] > Throwable #1: java.lang.AssertionError: Doc with id=1 not found in http://127.0.0.1:35191/collMinRf_1x3 due to: Path not found: /id; rsp={doc=null}
[junit4] > at __randomizedtesting.SeedInfo.seed([8E3A470CE083B308:66E78D64E7FDEF0]:0)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:603)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:558)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.testMinRf(HttpPartitionTest.java:249)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:127)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:564)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:844)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest_8E3A470CE083B308-001
[junit4] 2> Aug 21, 2017 1:38:43 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), a_t=PostingsFormat(name=LuceneVarGapFixedInterval), id=Lucene50(blocksize=128), text=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70), _version_=DocValuesFormat(name=Memory), range_facet_i_dv=DocValuesFormat(name=Direct), intDvoDefault=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Direct)}, maxPointsInLeafNode=1592, maxMBSortInHeap=6.100148947670907, sim=RandomSimilarity(queryNorm=true): {}, locale=en-ZW, timezone=Europe/Belgrade
[junit4] 2> NOTE: Linux 4.10.0-27-generic i386/Oracle Corporation 9 (32-bit)/cpus=8,threads=1,free=24063776,total=315621376
[junit4] 2> NOTE: All tests run in this JVM: [TestConfigSetProperties, TestPrepRecovery, DirectSolrConnectionTest, ZkControllerTest, QueryParsingTest, TestValueSourceCache, BadCopyFieldTest, MigrateRouteKeyTest, JsonLoaderTest, DataDrivenBlockJoinTest, DistributedMLTComponentTest, LoggingHandlerTest, CollectionsAPIDistributedZkTest, TestMultiValuedNumericRangeQuery, DistributedTermsComponentTest, ClusterStateUpdateTest, AnalysisAfterCoreReloadTest, MetricsConfigTest, TestDynamicFieldCollectionResource, SolrCmdDistributorTest, TestSolrCloudWithSecureImpersonation, TestDeleteCollectionOnDownNodes, TestSchemaNameResource, SolrSlf4jReporterTest, TestMaxTokenLenTokenizer, TestCoreContainer, TestFieldCacheReopen, TestExactSharedStatsCache, TolerantUpdateProcessorTest, TestGraphMLResponseWriter, AnalyticsMergeStrategyTest, TestAuthenticationFramework, UpdateRequestProcessorFactoryTest, SampleTest, TermVectorComponentTest, TestPartialUpdateDeduplication, DistributedVersionInfoTest, TestBlobHandler, SliceStateTest, TestBM25SimilarityFactory, MetricUtilsTest, HdfsLockFactoryTest, TestFastLRUCache, BitVectorTest, LukeRequestHandlerTest, TestLRUCache, SOLR749Test, HttpPartitionTest]
[junit4] Completed [141/729 (1!)] on J1 in 31.21s, 1 test, 1 failure <<< FAILURES!
[...truncated 38600 lines...]
[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_144) - Build # 296
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/296/
Java: 32bit/jdk1.8.0_144 -server -XX:+UseG1GC
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
7 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 1) Thread[id=1793, name=zkCallback-244-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1651, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 3) Thread[id=1649, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=1652, name=zkCallback-244-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1704, name=zkCallback-244-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1773, name=zkCallback-244-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1650, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-SendThread(127.0.0.1:41783), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:
1) Thread[id=1793, name=zkCallback-244-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1651, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
3) Thread[id=1649, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=1652, name=zkCallback-244-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=1704, name=zkCallback-244-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1773, name=zkCallback-244-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1650, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-SendThread(127.0.0.1:41783), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
at __randomizedtesting.SeedInfo.seed([14D04BAC0FDC6C15]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=1793, name=zkCallback-244-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1652, name=zkCallback-244-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1704, name=zkCallback-244-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=1773, name=zkCallback-244-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1650, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-SendThread(127.0.0.1:41783), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=1793, name=zkCallback-244-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1652, name=zkCallback-244-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1704, name=zkCallback-244-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=1773, name=zkCallback-244-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=1650, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-SendThread(127.0.0.1:41783), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
at __randomizedtesting.SeedInfo.seed([14D04BAC0FDC6C15]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test
Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:35085/collection1
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35085/collection1
at __randomizedtesting.SeedInfo.seed([14D04BAC0FDC6C15:9C847476A12001ED]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:638)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1583)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:213)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:526)
... 53 more
Build Log:
[...truncated 11188 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
[junit4] 2> 395344 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[14D04BAC0FDC6C15]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/init-core-data-001
[junit4] 2> 395344 WARN (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[14D04BAC0FDC6C15]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 395345 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[14D04BAC0FDC6C15]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 395346 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[14D04BAC0FDC6C15]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 395346 INFO (SUITE-ChaosMonkeyNothingIsSafeTest-seed#[14D04BAC0FDC6C15]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 395349 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 395349 INFO (Thread-311) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 395349 INFO (Thread-311) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 395353 ERROR (Thread-311) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 395449 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.ZkTestServer start zk server on port:41783
[junit4] 2> 395501 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 395502 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 395503 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 395503 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 395504 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 395504 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 395505 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 395505 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 395512 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 395513 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 395514 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractZkTestCase put /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 395514 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
[junit4] 2> 395595 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 395595 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@121f4ad{/,null,AVAILABLE}
[junit4] 2> 395597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@b88596{HTTP/1.1,[http/1.1]}{127.0.0.1:45253}
[junit4] 2> 395597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.Server Started @396957ms
[junit4] 2> 395597 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/tempDir-001/control/data, hostContext=/, hostPort=45253, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/control-001/cores}
[junit4] 2> 395604 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 395604 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 395604 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 395604 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 395604 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-21T06:40:52.292Z
[junit4] 2> 395610 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 395610 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/control-001/solr.xml
[junit4] 2> 395614 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 395616 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41783/solr
[junit4] 2> 395683 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:45253_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 395684 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:45253_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45253_
[junit4] 2> 395684 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:45253_ ] o.a.s.c.Overseer Overseer (id=98520114923962372-127.0.0.1:45253_-n_0000000000) starting
[junit4] 2> 395695 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:45253_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45253_
[junit4] 2> 395696 INFO (zkCallback-237-thread-1-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 395820 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:45253_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 395830 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:45253_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 395830 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:45253_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 395831 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:45253_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/control-001/cores
[junit4] 2> 395851 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 395851 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41783/solr ready
[junit4] 2> 395853 INFO (qtp9348697-1618) [n:127.0.0.1:45253_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45253_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 395857 INFO (OverseerThreadFactory-696-thread-1-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 395964 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 395965 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 396071 INFO (zkCallback-237-thread-1-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 396983 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 396990 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 397052 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 397059 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 397060 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 397060 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 397060 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 397064 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=64.3330078125, floorSegmentMB=1.3603515625, forceMergeDeletesPctAllowed=6.901292712528972, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5803029934945613
[junit4] 2> 397075 WARN (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 397115 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 397115 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 397115 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 397115 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 397117 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 397117 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1ec7f0b[control_collection_shard1_replica_n1] main]
[junit4] 2> 397118 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 397118 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 397119 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 397120 INFO (searcherExecutor-699-thread-1-processing-n:127.0.0.1:45253_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@1ec7f0b[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 397120 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576321840639377408
[junit4] 2> 397123 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 397123 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 397123 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45253/control_collection_shard1_replica_n1/
[junit4] 2> 397123 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 397123 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45253/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 397123 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 397125 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45253/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 397226 INFO (zkCallback-237-thread-1-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 397275 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 397276 INFO (qtp9348697-1614) [n:127.0.0.1:45253_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1312
[junit4] 2> 397278 INFO (qtp9348697-1618) [n:127.0.0.1:45253_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 397378 INFO (zkCallback-237-thread-1-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 397859 INFO (OverseerCollectionConfigSetProcessor-98520114923962372-127.0.0.1:45253_-n_0000000000) [n:127.0.0.1:45253_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 398278 INFO (qtp9348697-1618) [n:127.0.0.1:45253_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45253_&wt=javabin&version=2} status=0 QTime=2424
[junit4] 2> 398284 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 398285 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41783/solr ready
[junit4] 2> 398285 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 398286 INFO (qtp9348697-1618) [n:127.0.0.1:45253_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 398287 INFO (OverseerThreadFactory-696-thread-2-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 398288 WARN (OverseerThreadFactory-696-thread-2-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 398491 INFO (qtp9348697-1618) [n:127.0.0.1:45253_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 398491 INFO (qtp9348697-1618) [n:127.0.0.1:45253_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=204
[junit4] 2> 398613 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-1-001 of type TLOG
[junit4] 2> 398614 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 398615 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1091946{/,null,AVAILABLE}
[junit4] 2> 398615 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1d90a01{HTTP/1.1,[http/1.1]}{127.0.0.1:43325}
[junit4] 2> 398615 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.Server Started @399975ms
[junit4] 2> 398616 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=43325, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-1-001/cores}
[junit4] 2> 398616 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 398616 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 398616 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 398616 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 398616 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-21T06:40:55.304Z
[junit4] 2> 398621 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 398621 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-1-001/solr.xml
[junit4] 2> 398624 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 398626 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41783/solr
[junit4] 2> 398634 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:43325_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 398634 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:43325_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 398635 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:43325_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43325_
[junit4] 2> 398635 INFO (zkCallback-244-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 398635 INFO (zkCallback-237-thread-1-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 398636 INFO (zkCallback-249-thread-1-processing-n:127.0.0.1:43325_) [n:127.0.0.1:43325_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 398726 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:43325_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 398733 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:43325_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 398733 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:43325_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 398734 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:43325_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-1-001/cores
[junit4] 2> 398757 INFO (qtp14539996-1659) [n:127.0.0.1:43325_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:43325_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 398758 INFO (OverseerThreadFactory-696-thread-3-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:43325_ for creating new replica
[junit4] 2> 398758 INFO (OverseerCollectionConfigSetProcessor-98520114923962372-127.0.0.1:45253_-n_0000000000) [n:127.0.0.1:45253_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 398759 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 398760 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 399771 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 399788 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
[junit4] 2> 399873 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 399881 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
[junit4] 2> 399882 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 399882 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 399882 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
[junit4] 2> 399892 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=64.3330078125, floorSegmentMB=1.3603515625, forceMergeDeletesPctAllowed=6.901292712528972, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5803029934945613
[junit4] 2> 399908 WARN (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 399955 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 399955 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 399956 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 399956 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 399957 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 399958 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@fb84af[collection1_shard2_replica_t41] main]
[junit4] 2> 399958 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 399958 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 399959 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 399959 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576321843616284672
[junit4] 2> 399963 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 399963 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 399963 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43325/collection1_shard2_replica_t41/
[junit4] 2> 399963 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 399963 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:43325/collection1_shard2_replica_t41/ has no replicas
[junit4] 2> 399963 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 399963 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
[junit4] 2> 399964 INFO (searcherExecutor-710-thread-1-processing-n:127.0.0.1:43325_ x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@fb84af[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 399965 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:43325/collection1_shard2_replica_t41/ shard2
[junit4] 2> 400118 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 400119 INFO (qtp14539996-1661) [n:127.0.0.1:43325_ c:collection1 s:shard2 x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1359
[junit4] 2> 400120 INFO (qtp14539996-1659) [n:127.0.0.1:43325_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:43325_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1363
[junit4] 2> 400181 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-2-001 of type TLOG
[junit4] 2> 400181 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 400182 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@29a6c2{/,null,AVAILABLE}
[junit4] 2> 400182 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1222eeb{HTTP/1.1,[http/1.1]}{127.0.0.1:35085}
[junit4] 2> 400182 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.Server Started @401542ms
[junit4] 2> 400182 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=35085, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-2-001/cores}
[junit4] 2> 400182 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 400182 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 400182 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 400183 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 400183 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-21T06:40:56.871Z
[junit4] 2> 400190 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 400190 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-2-001/solr.xml
[junit4] 2> 400193 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 400196 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41783/solr
[junit4] 2> 400219 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:35085_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 400220 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:35085_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 400221 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:35085_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35085_
[junit4] 2> 400222 INFO (zkCallback-249-thread-1-processing-n:127.0.0.1:43325_) [n:127.0.0.1:43325_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 400222 INFO (zkCallback-244-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 400222 INFO (zkCallback-237-thread-2-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 400228 INFO (zkCallback-255-thread-1-processing-n:127.0.0.1:35085_) [n:127.0.0.1:35085_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 400360 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:35085_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 400371 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:35085_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 400371 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:35085_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 400372 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:35085_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-2-001/cores
[junit4] 2> 400404 INFO (qtp14539996-1660) [n:127.0.0.1:43325_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:35085_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 400405 INFO (OverseerCollectionConfigSetProcessor-98520114923962372-127.0.0.1:45253_-n_0000000000) [n:127.0.0.1:45253_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 400406 INFO (OverseerThreadFactory-696-thread-4-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:35085_ for creating new replica
[junit4] 2> 400409 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 400409 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 401417 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 401424 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
[junit4] 2> 401472 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 401480 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
[junit4] 2> 401480 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 401480 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 401480 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
[junit4] 2> 401483 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=64.3330078125, floorSegmentMB=1.3603515625, forceMergeDeletesPctAllowed=6.901292712528972, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5803029934945613
[junit4] 2> 401503 WARN (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 401542 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 401542 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 401544 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 401544 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 401545 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 401545 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@43e97[collection1_shard1_replica_t43] main]
[junit4] 2> 401552 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 401553 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 401554 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 401555 INFO (searcherExecutor-721-thread-1-processing-n:127.0.0.1:35085_ x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@43e97[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 401555 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576321845289811968
[junit4] 2> 401559 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 401559 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 401559 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35085/collection1_shard1_replica_t43/
[junit4] 2> 401559 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 401559 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:35085/collection1_shard1_replica_t43/ has no replicas
[junit4] 2> 401559 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 401559 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
[junit4] 2> 401561 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35085/collection1_shard1_replica_t43/ shard1
[junit4] 2> 401711 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 401712 INFO (qtp31447678-1689) [n:127.0.0.1:35085_ c:collection1 s:shard1 x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1303
[junit4] 2> 401714 INFO (qtp14539996-1660) [n:127.0.0.1:43325_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:35085_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1309
[junit4] 2> 401791 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-3-001 of type TLOG
[junit4] 2> 401791 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 401792 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@9798c6{/,null,AVAILABLE}
[junit4] 2> 401792 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@562377{HTTP/1.1,[http/1.1]}{127.0.0.1:36263}
[junit4] 2> 401792 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.e.j.s.Server Started @403152ms
[junit4] 2> 401792 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=36263, coreRootDirectory=/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-3-001/cores}
[junit4] 2> 401792 ERROR (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 401793 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.1.0
[junit4] 2> 401793 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 401793 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null, Default config dir: null
[junit4] 2> 401793 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-08-21T06:40:58.481Z
[junit4] 2> 401796 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 401796 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-3-001/solr.xml
[junit4] 2> 401799 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 401805 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41783/solr
[junit4] 2> 401820 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:36263_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 401821 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:36263_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 401822 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:36263_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36263_
[junit4] 2> 401822 INFO (zkCallback-249-thread-2-processing-n:127.0.0.1:43325_) [n:127.0.0.1:43325_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 401822 INFO (zkCallback-244-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 401822 INFO (zkCallback-255-thread-2-processing-n:127.0.0.1:35085_) [n:127.0.0.1:35085_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 401823 INFO (zkCallback-237-thread-1-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 401828 INFO (zkCallback-261-thread-1-processing-n:127.0.0.1:36263_) [n:127.0.0.1:36263_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 401868 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:36263_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 401919 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:36263_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 401919 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:36263_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 401921 INFO (TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]) [n:127.0.0.1:36263_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-3-001/cores
[junit4] 2> 401950 INFO (qtp14539996-1655) [n:127.0.0.1:43325_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:36263_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 401952 INFO (OverseerThreadFactory-696-thread-5-processing-n:127.0.0.1:45253_) [n:127.0.0.1:45253_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:36263_ for creating new replica
[junit4] 2> 401952 INFO (OverseerCollectionConfigSetProcessor-98520114923962372-127.0.0.1:45253_-n_0000000000) [n:127.0.0.1:45253_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 401954 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
[junit4] 2> 401955 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 402970 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.1.0
[junit4] 2> 402984 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema [collection1_shard2_replica_t45] Schema name=test
[junit4] 2> 403029 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 403035 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t45' using configuration from collection collection1, trusted=true
[junit4] 2> 403036 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t45' (registry 'solr.core.collection1.shard2.replica_t45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ab750b
[junit4] 2> 403036 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 403036 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [[collection1_shard2_replica_t45] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-3-001/cores/collection1_shard2_replica_t45], dataDir=[/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest_14D04BAC0FDC6C15-001/shard-3-001/cores/collection1_shard2_replica_t45/data/]
[junit4] 2> 403038 INFO (qtp18856748-1721) [n:127.0.0.1:36263_ c:collection1 s:shard2 x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=64.3330078125, floorSegmentMB=1.3603515625, forceMergeDeletesPctAllowed=6.901292712528972, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5803029934945613
[jun
[...truncated too long message...]
ronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=1651, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
[junit4] > 3) Thread[id=1649, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=1652, name=zkCallback-244-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=1704, name=zkCallback-244-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=1773, name=zkCallback-244-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=1650, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-SendThread(127.0.0.1:41783), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
[junit4] > at __randomizedtesting.SeedInfo.seed([14D04BAC0FDC6C15]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=1793, name=zkCallback-244-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 2) Thread[id=1652, name=zkCallback-244-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 3) Thread[id=1704, name=zkCallback-244-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=1773, name=zkCallback-244-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=1650, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[14D04BAC0FDC6C15]-SendThread(127.0.0.1:41783), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > at __randomizedtesting.SeedInfo.seed([14D04BAC0FDC6C15]:0)
[junit4] Completed [124/729 (1!)] on J0 in 75.91s, 1 test, 3 errors <<< FAILURES!
[...truncated 41337 lines...]