You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2016/11/25 20:57:30 UTC
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 514 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/514/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.update.AutoCommitTest.testMaxTime
Error Message:
Exception during query
Stack Trace:
java.lang.RuntimeException: Exception during query
at __randomizedtesting.SeedInfo.seed([2FEDEB1389942E77:B51996F1170EB24B]:0)
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:812)
at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>
request was:q=id:529&qt=standard&start=0&rows=20&version=2.2
at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:805)
... 40 more
Build Log:
[...truncated 10771 lines...]
[junit4] Suite: org.apache.solr.update.AutoCommitTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_2FEDEB1389942E77-001/init-core-data-001
[junit4] 2> 79811 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 79811 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 79812 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 79838 WARN (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 79840 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 79858 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=test
[junit4] 2> 79924 WARN (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 79925 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 79934 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
[junit4] 2> 79936 INFO (coreLoadExecutor-693-thread-1) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 79958 WARN (coreLoadExecutor-693-thread-1) [ ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 79959 INFO (coreLoadExecutor-693-thread-1) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 79976 INFO (coreLoadExecutor-693-thread-1) [ ] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 80033 WARN (coreLoadExecutor-693-thread-1) [ ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 80035 INFO (coreLoadExecutor-693-thread-1) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 80041 INFO (coreLoadExecutor-693-thread-1) [ ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1
[junit4] 2> 80042 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_2FEDEB1389942E77-001/init-core-data-001/]
[junit4] 2> 80042 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@609fdd0
[junit4] 2> 80043 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=6.0595703125, floorSegmentMB=0.3271484375, forceMergeDeletesPctAllowed=22.115973098993962, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.41869213622183066
[junit4] 2> 80054 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 80054 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 80055 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 80055 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 80055 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=92.955078125, floorSegmentMB=1.4931640625, forceMergeDeletesPctAllowed=15.302263893917784, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6048433566718587
[junit4] 2> 80055 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4e3d0a34[collection1] main]
[junit4] 2> 80056 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 80060 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 80078 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 80078 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 80079 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 80079 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 80079 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 80079 INFO (coreLoadExecutor-693-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1552001053394731008
[junit4] 2> 80079 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 80080 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 80080 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 80081 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 80081 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 80081 INFO (searcherExecutor-694-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4e3d0a34[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 80082 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 80084 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testMaxDocs
[junit4] 2> 80088 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{deleteByQuery=*:* (-1552001053399973888)} 0 3
[junit4] 2> 80090 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 80115 WARN (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 80116 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 80133 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 80193 WARN (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 80195 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 80201 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1
[junit4] 2> 80202 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_2FEDEB1389942E77-001/init-core-data-001/]
[junit4] 2> 80202 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@609fdd0
[junit4] 2> 80217 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 80217 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 80217 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4f990f73[collection1] main]
[junit4] 2> 80217 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 80218 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 80221 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 80221 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 80222 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 80222 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 80222 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 80222 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 80223 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 80223 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 80223 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 80224 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 80225 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4f990f73[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 80225 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=58.802734375, floorSegmentMB=1.1865234375, forceMergeDeletesPctAllowed=0.9261852276991367, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.548538173004159
[junit4] 2> 80226 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 80226 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d2f5c67[collection1] main]
[junit4] 2> 80226 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@5d0b8bb
[junit4] 2> 80227 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2d2f5c67[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 80236 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[0 (1552001053558308864)]} 0 1
[junit4] 2> 80237 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[1 (1552001053560406016)]} 0 2
[junit4] 2> 80237 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[2 (1552001053560406017)]} 0 2
[junit4] 2> 80238 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[3 (1552001053560406018)]} 0 2
[junit4] 2> 80238 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[4 (1552001053561454592)]} 0 2
[junit4] 2> 80238 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[5 (1552001053561454593)]} 0 3
[junit4] 2> 80238 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[6 (1552001053561454594)]} 0 3
[junit4] 2> 80239 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[7 (1552001053561454595)]} 0 3
[junit4] 2> 80239 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[8 (1552001053562503168)]} 0 4
[junit4] 2> 80240 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[9 (1552001053562503169)]} 0 5
[junit4] 2> 80240 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[10 (1552001053563551744)]} 0 5
[junit4] 2> 80240 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[11 (1552001053563551745)]} 0 5
[junit4] 2> 80241 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[12 (1552001053564600320)]} 0 5
[junit4] 2> 80241 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[13 (1552001053564600321)]} 0 6
[junit4] 2> 80241 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:1&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 80242 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[14 (1552001053565648896)]} 0 7
[junit4] 2> 80243 INFO (commitScheduler-702-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 80266 INFO (commitScheduler-702-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@62289006[collection1] main]
[junit4] 2> 80266 INFO (commitScheduler-702-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 80268 INFO (searcherExecutor-699-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@62289006[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C15)))}
[junit4] 2> 80494 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[15 (1552001053829890048)]} 0 259
[junit4] 2> 80495 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:14&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 80497 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:15&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 80497 INFO (TEST-AutoCommitTest.testMaxDocs-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxDocs
[junit4] 2> 80503 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testMaxTime
[junit4] 2> 80509 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1de3db0[collection1] realtime]
[junit4] 2> 80509 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{deleteByQuery=*:* (-1552001053839327232)} 0 6
[junit4] 2> 80510 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 80539 WARN (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 80541 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 80559 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 80623 WARN (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 80624 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 80633 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1
[junit4] 2> 80634 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_2FEDEB1389942E77-001/init-core-data-001/]
[junit4] 2> 80634 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@609fdd0
[junit4] 2> 80653 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 80653 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 80653 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@314753bd[collection1] main]
[junit4] 2> 80653 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 80655 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 80661 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 80666 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 80667 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 80667 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 80667 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 80667 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 80668 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 80668 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 80668 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 80669 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 80669 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@314753bd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 80670 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15452307610570504]
[junit4] 2> 80670 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 80670 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1a8115ad[collection1] main]
[junit4] 2> 80670 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@6e4f733e
[junit4] 2> 80679 INFO (searcherExecutor-703-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1a8115ad[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 80680 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1552001054022828032)]} 0 1
[junit4] 2> 80680 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 80681 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1552001054024925184)]} 0 2
[junit4] 2> 80681 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 80681 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
[junit4] 2> xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] 2> <response>
[junit4] 2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
[junit4] 2> </response>
[junit4] 2>
[junit4] 2> request was:q=id:529&qt=standard&start=0&rows=20&version=2.2
[junit4] 2> 80681 ERROR (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=id:529&qt=standard&start=0&rows=20&version=2.2:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
[junit4] 2> xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] 2> <response>
[junit4] 2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
[junit4] 2> </response>
[junit4] 2>
[junit4] 2> request was:q=id:529&qt=standard&start=0&rows=20&version=2.2
[junit4] 2> at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:805)
[junit4] 2> at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:498)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 80682 INFO (TEST-AutoCommitTest.testMaxTime-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxTime
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=AutoCommitTest -Dtests.method=testMaxTime -Dtests.seed=2FEDEB1389942E77 -Dtests.slow=true -Dtests.locale=uk-UA -Dtests.timezone=MST7MDT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.19s J1 | AutoCommitTest.testMaxTime <<<
[junit4] > Throwable #1: java.lang.RuntimeException: Exception during query
[junit4] > at __randomizedtesting.SeedInfo.seed([2FEDEB1389942E77:B51996F1170EB24B]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:812)
[junit4] > at org.apache.solr.update.AutoCommitTest.testMaxTime(AutoCommitTest.java:270)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
[junit4] > xml response was: <?xml version="1.0" encoding="UTF-8"?>
[junit4] > <response>
[junit4] > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
[junit4] > </response>
[junit4] > request was:q=id:529&qt=standard&start=0&rows=20&version=2.2
[junit4] > at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:805)
[junit4] > ... 40 more
[junit4] 2> 80691 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testCommitWithin
[junit4] 2> 80695 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5a57dfb3[collection1] realtime]
[junit4] 2> 80695 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{deleteByQuery=*:* (-1552001054036459520)} 0 4
[junit4] 2> 80696 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 80722 WARN (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
[junit4] 2> 80724 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 80741 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 80802 WARN (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
[junit4] 2> 80804 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 80811 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1
[junit4] 2> 80811 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_2FEDEB1389942E77-001/init-core-data-001/]
[junit4] 2> 80812 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@609fdd0
[junit4] 2> 80827 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 80827 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 80828 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@26cde388[collection1] main]
[junit4] 2> 80828 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 80829 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 80833 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 80836 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=558489450, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 80836 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
[junit4] 2> 80837 WARN (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrCore [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
[junit4] 2> 80837 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@71374cd5[collection1] main]
[junit4] 2> 80837 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@4fb13131
[junit4] 2> 80838 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 80839 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 80839 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 80839 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 80839 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 80840 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 80840 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 80840 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 80841 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 80841 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@26cde388[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 80841 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 80842 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 80842 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 80842 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 80842 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 80843 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 80844 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 80844 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 80844 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 80845 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@71374cd5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 80849 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1552001054201085952)]} 0 1
[junit4] 2> 80849 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 84854 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 84859 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@65ae186a[collection1] main]
[junit4] 2> 84859 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 84860 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@65ae186a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(6.4.0):C1)))}
[junit4] 2> 84888 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1552001058436284416)]} 0 4042
[junit4] 2> 84888 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 84889 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[529 (-1552001058438381568)]} 0 4043
[junit4] 2> 84889 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 85893 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 85897 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e48e9c6[collection1] main]
[junit4] 2> 85897 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 85898 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5e48e9c6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(6.4.0):C1)))}
[junit4] 2> 85899 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 85899 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 85900 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059498491904)]} 0 5054
[junit4] 2> 85901 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059499540480)]} 0 5055
[junit4] 2> 85901 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059499540481)]} 0 5055
[junit4] 2> 85901 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059499540482)]} 0 5055
[junit4] 2> 85901 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059499540483)]} 0 5056
[junit4] 2> 85901 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059499540484)]} 0 5056
[junit4] 2> 85902 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059500589056)]} 0 5056
[junit4] 2> 85902 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059500589057)]} 0 5056
[junit4] 2> 85902 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059500589058)]} 0 5056
[junit4] 2> 85902 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1552001059500589059)]} 0 5056
[junit4] 2> 85902 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:500&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 85903 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059501637632)]} 0 5057
[junit4] 2> 85903 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059501637633)]} 0 5057
[junit4] 2> 85903 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059501637634)]} 0 5057
[junit4] 2> 85903 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059501637635)]} 0 5057
[junit4] 2> 85903 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059501637636)]} 0 5058
[junit4] 2> 85904 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059502686208)]} 0 5059
[junit4] 2> 85905 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059503734784)]} 0 5059
[junit4] 2> 85905 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059503734785)]} 0 5059
[junit4] 2> 85905 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059503734786)]} 0 5059
[junit4] 2> 85905 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[530 (-1552001059503734787)]} 0 5059
[junit4] 2> 85905 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 86906 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 86909 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@62a6053d[collection1] main]
[junit4] 2> 86910 INFO (commitScheduler-710-thread-1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 86911 INFO (searcherExecutor-707-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@62a6053d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(6.4.0):C10/9:delGen=1)))}
[junit4] 2> 87158 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:500&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
[junit4] 2> 87159 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=null path=null params={q=id:530&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
[junit4] 2> 87159 INFO (TEST-AutoCommitTest.testCommitWithin-seed#[2FEDEB1389942E77]) [ x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testCommitWithin
[junit4] 2> 87160 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 87160 INFO (SUITE-AutoCommitTest-seed#[2FEDEB1389942E77]-worker) [ x:collection1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1494694321
[junit4] 2> 87160 INFO (coreCloseExecutor-711-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@5523dab1
[junit4] 2> 87161 INFO (coreCloseExecutor-711-thread-1-processing-x:collection1) [ x:collection1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
[junit4] 2> 87161 INFO (coreCloseExecutor-711-thread-1-processing-x:collection1) [ x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@2e3206b
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.update.AutoCommitTest_2FEDEB1389942E77-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {range_facet_l_dv=Lucene50(blocksize=128), _version_=BlockTreeOrds(blocksize=128), field_t=FSTOrd50, multiDefault=PostingsFormat(name=Memory doPackFST= true), subject=PostingsFormat(name=Memory doPackFST= true), intDefault=BlockTreeOrds(blocksize=128), id=Lucene50(blocksize=128), range_facet_i_dv=BlockTreeOrds(blocksize=128), text=FSTOrd50, range_facet_l=BlockTreeOrds(blocksize=128), timestamp=BlockTreeOrds(blocksize=128)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=133, maxMBSortInHeap=5.340051923003928, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=uk-UA, timezone=MST7MDT
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=216814656,total=318767104
[junit4] 2> NOTE: All tests run in this JVM: [TestJavabinTupleStreamParser, TestInitQParser, SpellPossibilityIteratorTest, TestTrieFacet, CreateCollectionCleanupTest, UUIDFieldTest, CSVRequestHandlerTest, AnalyticsQueryTest, AtomicUpdatesTest, TestCoreDiscovery, TestDistributedMissingSort, TestSolrCloudWithSecureImpersonation, TriLevelCompositeIdRoutingTest, TestSolrDynamicMBean, TestRecovery, TestStandardQParsers, AutoCommitTest]
[junit4] Completed [32/657 (1!)] on J1 in 7.38s, 3 tests, 1 error <<< FAILURES!
[...truncated 62801 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 518 -
Failure!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/518/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
2 tests failed.
FAILED: org.apache.solr.cloud.RecoveryZkTest.test
Error Message:
Error from server at https://127.0.0.1:35281/solr: Cannot create collection recoverytest. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 1. This allows a maximum of 1 to be created. Value of numShards is 1 and value of replicationFactor is 2. This requires 2 shards to be created (higher than the allowed number)
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:35281/solr: Cannot create collection recoverytest. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 1. This allows a maximum of 1 to be created. Value of numShards is 1 and value of replicationFactor is 2. This requires 2 shards to be created (higher than the allowed number)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:593)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:435)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:387)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1340)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1091)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1033)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:68)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor161.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:705) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:767) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1006) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:871) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:775) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor161.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:705)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:767)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1006)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:871)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:775)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([8B11045FDE87FCC5]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:260)
at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 11226 lines...]
[junit4] Suite: org.apache.solr.cloud.RecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_8B11045FDE87FCC5-001/init-core-data-001
[junit4] 2> 722886 INFO (SUITE-RecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 722887 INFO (SUITE-RecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_8B11045FDE87FCC5-001/tempDir-001
[junit4] 2> 722888 INFO (SUITE-RecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 722888 INFO (Thread-1242) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 722888 INFO (Thread-1242) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 722988 INFO (SUITE-RecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:34726
[junit4] 2> 723006 INFO (jetty-launcher-687-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 723008 INFO (jetty-launcher-687-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7e50e879{/solr,null,AVAILABLE}
[junit4] 2> 723011 INFO (jetty-launcher-687-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7d32a5dd{SSL,[ssl, http/1.1]}{127.0.0.1:59078}
[junit4] 2> 723011 INFO (jetty-launcher-687-thread-1) [ ] o.e.j.s.Server Started @725880ms
[junit4] 2> 723011 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59078}
[junit4] 2> 723011 ERROR (jetty-launcher-687-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 723011 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 723011 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 723011 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 723011 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-27T14:27:44.190Z
[junit4] 2> 723015 INFO (jetty-launcher-687-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 723016 INFO (jetty-launcher-687-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@52241a60{/solr,null,AVAILABLE}
[junit4] 2> 723018 INFO (jetty-launcher-687-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7bec9988{SSL,[ssl, http/1.1]}{127.0.0.1:35281}
[junit4] 2> 723018 INFO (jetty-launcher-687-thread-2) [ ] o.e.j.s.Server Started @725888ms
[junit4] 2> 723018 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35281}
[junit4] 2> 723018 ERROR (jetty-launcher-687-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 723019 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 723019 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 723019 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 723019 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-27T14:27:44.198Z
[junit4] 2> 723028 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 723240 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 723252 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 723253 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34726/solr
[junit4] 2> 723286 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 723286 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34726/solr
[junit4] 2> 723314 INFO (jetty-launcher-687-thread-1) [n:127.0.0.1:59078_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59078_solr
[junit4] 2> 723317 INFO (jetty-launcher-687-thread-1) [n:127.0.0.1:59078_solr ] o.a.s.c.Overseer Overseer (id=97010113843757060-127.0.0.1:59078_solr-n_0000000000) starting
[junit4] 2> 723325 INFO (jetty-launcher-687-thread-1) [n:127.0.0.1:59078_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59078_solr
[junit4] 2> 723395 INFO (jetty-launcher-687-thread-1) [n:127.0.0.1:59078_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_8B11045FDE87FCC5-001/tempDir-001/node1/.
[junit4] 2> 723423 INFO (jetty-launcher-687-thread-2) [n:127.0.0.1:35281_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 723429 INFO (jetty-launcher-687-thread-2) [n:127.0.0.1:35281_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35281_solr
[junit4] 2> 723510 INFO (jetty-launcher-687-thread-2) [n:127.0.0.1:35281_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_8B11045FDE87FCC5-001/tempDir-001/node2/.
[junit4] 2> 723518 INFO (zkCallback-694-thread-1-processing-n:127.0.0.1:59078_solr) [n:127.0.0.1:59078_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 723528 INFO (SUITE-RecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 723529 INFO (SUITE-RecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34726/solr ready
[junit4] 2> 723535 INFO (zkCallback-699-thread-1-processing-n:127.0.0.1:35281_solr) [n:127.0.0.1:35281_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 723550 INFO (TEST-RecoveryZkTest.test-seed#[8B11045FDE87FCC5]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 723603 INFO (qtp1955414459-4683) [n:127.0.0.1:35281_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 723610 INFO (OverseerThreadFactory-2129-thread-1-processing-n:127.0.0.1:59078_solr) [n:127.0.0.1:59078_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 723610 WARN (OverseerThreadFactory-2129-thread-1-processing-n:127.0.0.1:59078_solr) [n:127.0.0.1:59078_solr ] o.a.s.c.CreateCollectionCmd Specified replicationFactor of 2 on collection recoverytest is higher than or equal to the number of Solr instances currently live or live and part of your createNodeSet(1). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 723611 ERROR (OverseerThreadFactory-2129-thread-1-processing-n:127.0.0.1:59078_solr) [n:127.0.0.1:59078_solr ] o.a.s.c.OverseerCollectionMessageHandler Collection: recoverytest operation: create failed:org.apache.solr.common.SolrException: Cannot create collection recoverytest. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 1. This allows a maximum of 1 to be created. Value of numShards is 1 and value of replicationFactor is 2. This requires 2 shards to be created (higher than the allowed number)
[junit4] 2> at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:148)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:224)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:463)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2>
[junit4] 2> 723645 INFO (qtp1955414459-4683) [n:127.0.0.1:35281_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=400 QTime=41
[junit4] 2> 723648 INFO (TEST-RecoveryZkTest.test-seed#[8B11045FDE87FCC5]) [ ] o.a.s.SolrTestCaseJ4 ###Ending test
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=8B11045FDE87FCC5 -Dtests.slow=true -Dtests.locale=ar -Dtests.timezone=Atlantic/Madeira -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.12s J0 | RecoveryZkTest.test <<<
[junit4] > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:35281/solr: Cannot create collection recoverytest. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 1. This allows a maximum of 1 to be created. Value of numShards is 1 and value of replicationFactor is 2. This requires 2 shards to be created (higher than the allowed number)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:593)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:435)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:387)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1340)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1091)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1033)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
[junit4] > at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:68)
[junit4] > at java.lang.Thread.run(Thread.java:745)Throwable #2: java.lang.NullPointerException
[junit4] > at org.apache.solr.cloud.RecoveryZkTest.stopThreads(RecoveryZkTest.java:55)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 723658 INFO (jetty-launcher-687-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@7d32a5dd{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 723658 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=101617813
[junit4] 2> 723662 INFO (jetty-launcher-687-thread-2) [ ] o.a.s.c.Overseer Overseer (id=97010113843757060-127.0.0.1:59078_solr-n_0000000000) closing
[junit4] 2> 723663 INFO (jetty-launcher-687-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@7bec9988{SSL,[ssl, http/1.1]}{127.0.0.1:0}
[junit4] 2> 723664 INFO (jetty-launcher-687-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=674466636
[junit4] 2> 723672 INFO (jetty-launcher-687-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@52241a60{/solr,null,UNAVAILABLE}
[junit4] 2> 725183 WARN (zkCallback-694-thread-2-processing-n:127.0.0.1:59078_solr) [n:127.0.0.1:59078_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 725184 WARN (OverseerStateUpdate-97010113843757060-127.0.0.1:59078_solr-n_0000000000) [n:127.0.0.1:59078_solr ] o.a.s.c.Overseer Solr cannot talk to ZK, exiting Overseer work queue loop
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /live_nodes
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:339)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:336)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:336)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.refreshLiveNodes(ZkStateReader.java:664)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:268)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:140)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 725184 INFO (OverseerStateUpdate-97010113843757060-127.0.0.1:59078_solr-n_0000000000) [n:127.0.0.1:59078_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:59078_solr
[junit4] 2> 725184 INFO (zkCallback-694-thread-1-processing-n:127.0.0.1:59078_solr) [n:127.0.0.1:59078_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 725185 INFO (jetty-launcher-687-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7e50e879{/solr,null,UNAVAILABLE}
[junit4] 2> 725194 INFO (SUITE-RecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34726 34726
[junit4] 2> 725219 INFO (Thread-1242) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34726 34726
[junit4] 2> 725220 WARN (Thread-1242) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2>
[junit4] 2> 725221 INFO (SUITE-RecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest_8B11045FDE87FCC5-001
[junit4] 2> Nov 27, 2016 2:27:46 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=455, maxMBSortInHeap=7.682942685719383, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=ar, timezone=Atlantic/Madeira
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=86406184,total=261984256
[junit4] 2> NOTE: All tests run in this JVM: [TestPKIAuthenticationPlugin, TestDocumentBuilder, ResponseHeaderTest, TestShardHandlerFactory, TestOrdValues, TestSolrConfigHandlerConcurrent, IndexBasedSpellCheckerTest, TestRandomRequestDistribution, TestManagedSchemaAPI, TestSolrCloudSnapshots, TestConfigReload, TestFieldCacheVsDocValues, HdfsUnloadDistributedZkTest, TestPhraseSuggestions, VersionInfoTest, ShardRoutingTest, TestHashPartitioner, TestCustomDocTransformer, HighlighterTest, CollectionsAPIAsyncDistributedZkTest, TestUpdate, TestComplexPhraseQParserPlugin, TestTrackingShardHandlerFactory, HardAutoCommitTest, RestartWhileUpdatingTest, TestUseDocValuesAsStored2, TestStressReorder, TestComponentsName, TestConfigOverlay, HdfsCollectionsAPIDistributedZkTest, CdcrUpdateLogTest, TestSchemaNameResource, SolrCoreTest, SampleTest, TestLeaderElectionWithEmptyReplica, HttpPartitionTest, TestCryptoKeys, TestDistributedSearch, DistribDocExpirationUpdateProcessorTest, PrimitiveFieldTypeTest, TestSolrIndexConfig, CurrencyFieldXmlFileTest, SpatialHeatmapFacetsTest, TestLeaderInitiatedRecoveryThread, ReplicaListTransformerTest, TestStressRecovery, TestOverriddenPrefixQueryForCustomFieldType, ResponseLogComponentTest, BJQParserTest, TestXmlQParserPlugin, RecoveryZkTest]
[junit4] Completed [170/658 (1!)] on J0 in 3.25s, 1 test, 1 error <<< FAILURES!
[...truncated 1096 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8B11045FDE87FCC5-001/init-core-data-001
[junit4] 2> 2423136 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2423137 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8B11045FDE87FCC5-001/tempDir-001
[junit4] 2> 2423137 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2423137 INFO (Thread-6255) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2423137 INFO (Thread-6255) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2423237 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:33037
[junit4] 2> 2423250 INFO (jetty-launcher-10861-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2423252 INFO (jetty-launcher-10861-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3a606bd4{/solr,null,AVAILABLE}
[junit4] 2> 2423250 INFO (jetty-launcher-10861-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2423257 INFO (jetty-launcher-10861-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@17aa2228{/solr,null,AVAILABLE}
[junit4] 2> 2423257 INFO (jetty-launcher-10861-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@11094a56{SSL,[ssl, http/1.1]}{127.0.0.1:46673}
[junit4] 2> 2423257 INFO (jetty-launcher-10861-thread-1) [ ] o.e.j.s.Server Started @2426868ms
[junit4] 2> 2423257 INFO (jetty-launcher-10861-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46673}
[junit4] 2> 2423257 ERROR (jetty-launcher-10861-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2423257 INFO (jetty-launcher-10861-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2423257 INFO (jetty-launcher-10861-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2423258 INFO (jetty-launcher-10861-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2423258 INFO (jetty-launcher-10861-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-27T14:56:04.437Z
[junit4] 2> 2423258 INFO (jetty-launcher-10861-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@46ea1f5c{SSL,[ssl, http/1.1]}{127.0.0.1:48362}
[junit4] 2> 2423258 INFO (jetty-launcher-10861-thread-2) [ ] o.e.j.s.Server Started @2426870ms
[junit4] 2> 2423258 INFO (jetty-launcher-10861-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48362}
[junit4] 2> 2423258 ERROR (jetty-launcher-10861-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2423259 INFO (jetty-launcher-10861-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2423259 INFO (jetty-launcher-10861-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2423259 INFO (jetty-launcher-10861-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2423259 INFO (jetty-launcher-10861-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-27T14:56:04.438Z
[junit4] 2> 2423262 INFO (jetty-launcher-10861-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2423262 INFO (jetty-launcher-10861-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2423270 INFO (jetty-launcher-10861-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2423270 INFO (jetty-launcher-10861-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33037/solr
[junit4] 2> 2423271 INFO (jetty-launcher-10861-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2423272 INFO (jetty-launcher-10861-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33037/solr
[junit4] 2> 2423307 INFO (jetty-launcher-10861-thread-1) [n:127.0.0.1:46673_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46673_solr
[junit4] 2> 2423308 INFO (jetty-launcher-10861-thread-2) [n:127.0.0.1:48362_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48362_solr
[junit4] 2> 2423310 INFO (jetty-launcher-10861-thread-1) [n:127.0.0.1:46673_solr ] o.a.s.c.Overseer Overseer (id=97010225271144454-127.0.0.1:46673_solr-n_0000000000) starting
[junit4] 2> 2423323 INFO (jetty-launcher-10861-thread-1) [n:127.0.0.1:46673_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46673_solr
[junit4] 2> 2423380 INFO (zkCallback-10871-thread-1-processing-n:127.0.0.1:48362_solr) [n:127.0.0.1:48362_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2423394 INFO (zkCallback-10872-thread-1-processing-n:127.0.0.1:46673_solr) [n:127.0.0.1:46673_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2423445 INFO (zkCallback-10871-thread-1-processing-n:127.0.0.1:48362_solr) [n:127.0.0.1:48362_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2423489 INFO (OverseerStateUpdate-97010225271144454-127.0.0.1:46673_solr-n_0000000000) [n:127.0.0.1:46673_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2423515 INFO (jetty-launcher-10861-thread-1) [n:127.0.0.1:46673_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8B11045FDE87FCC5-001/tempDir-001/node1/.
[junit4] 2> 2423593 INFO (jetty-launcher-10861-thread-2) [n:127.0.0.1:48362_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8B11045FDE87FCC5-001/tempDir-001/node2/.
[junit4] 2> 2423601 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2423603 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33037/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 2423640 WARN (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 2423649 WARN (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2423651 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2423665 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_46060_hdfs____zb6w1m/webapp
[junit4] 2> 2424216 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:46060
[junit4] 2> 2424345 WARN (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2424346 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2424360 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_41896_datanode____src61r/webapp
[junit4] 2> 2424820 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41896
[junit4] 2> 2424895 WARN (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 2424896 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 2424912 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_64880_datanode____emwjo9/webapp
[junit4] 2> 2424957 INFO (IPC Server handler 2 on 52615) [ ] BlockStateChange BLOCK* processReport: from storage DS-38811a91-dbc5-4f0e-9d5f-96d3c1408fcb node DatanodeRegistration(127.0.0.1:58758, datanodeUuid=0e41dbbf-963c-4d5a-89e1-a407734c9ee9, infoPort=62601, infoSecurePort=0, ipcPort=36459, storageInfo=lv=-56;cid=testClusterID;nsid=1862498236;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2424957 INFO (IPC Server handler 2 on 52615) [ ] BlockStateChange BLOCK* processReport: from storage DS-31a6424c-08ad-4f03-8663-73d81dac5516 node DatanodeRegistration(127.0.0.1:58758, datanodeUuid=0e41dbbf-963c-4d5a-89e1-a407734c9ee9, infoPort=62601, infoSecurePort=0, ipcPort=36459, storageInfo=lv=-56;cid=testClusterID;nsid=1862498236;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2425453 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:64880
[junit4] 2> 2426030 INFO (IPC Server handler 5 on 52615) [ ] BlockStateChange BLOCK* processReport: from storage DS-165617a1-5bfd-453b-9219-8edf920eacd9 node DatanodeRegistration(127.0.0.1:38289, datanodeUuid=c71bb449-fb2e-4319-96f7-fa7381210cd9, infoPort=43425, infoSecurePort=0, ipcPort=59954, storageInfo=lv=-56;cid=testClusterID;nsid=1862498236;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 2426030 INFO (IPC Server handler 5 on 52615) [ ] BlockStateChange BLOCK* processReport: from storage DS-20d5653e-c31a-4421-acca-7ef2c5693694 node DatanodeRegistration(127.0.0.1:38289, datanodeUuid=c71bb449-fb2e-4319-96f7-fa7381210cd9, infoPort=43425, infoSecurePort=0, ipcPort=59954, storageInfo=lv=-56;cid=testClusterID;nsid=1862498236;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 2426170 INFO (TEST-HdfsRecoveryZkTest.test-seed#[8B11045FDE87FCC5]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 2426215 INFO (qtp2127796789-48839) [n:127.0.0.1:46673_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2426219 INFO (OverseerThreadFactory-9711-thread-1-processing-n:127.0.0.1:46673_solr) [n:127.0.0.1:46673_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 2426344 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2426359 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 2426466 INFO (zkCallback-10871-thread-1-processing-n:127.0.0.1:48362_solr) [n:127.0.0.1:48362_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2426466 INFO (zkCallback-10872-thread-1-processing-n:127.0.0.1:46673_solr) [n:127.0.0.1:46673_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2427357 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2427366 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 2427369 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2427370 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 2427370 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 2427377 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:52615/data
[junit4] 2> 2427377 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2427377 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8B11045FDE87FCC5-001/tempDir-001/node1/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:52615/data/recoverytest/core_node1/data/]
[junit4] 2> 2427377 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:52615/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 2427379 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 2427381 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2427382 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 2427383 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:52615/data
[junit4] 2> 2427383 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 2427383 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_8B11045FDE87FCC5-001/tempDir-001/node2/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:52615/data/recoverytest/core_node2/data/]
[junit4] 2> 2427383 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:52615/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 2427386 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2427387 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2427387 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new global HDFS BlockCache
[junit4] 2> 2427390 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2427390 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2427398 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2427399 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2427400 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:52615/data/recoverytest/core_node1/data
[junit4] 2> 2427400 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:52615/data/recoverytest/core_node2/data
[junit4] 2> 2427421 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:52615/data/recoverytest/core_node1/data/index
[junit4] 2> 2427429 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2427429 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2427434 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2427437 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:52615/data/recoverytest/core_node2/data/index
[junit4] 2> 2427446 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 2427446 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 2427453 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 2427462 INFO (IPC Server handler 8 on 52615) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:58758 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-165617a1-5bfd-453b-9219-8edf920eacd9:NORMAL:127.0.0.1:38289|RBW], ReplicaUC[[DISK]DS-38811a91-dbc5-4f0e-9d5f-96d3c1408fcb:NORMAL:127.0.0.1:58758|RBW]]} size 0
[junit4] 2> 2427465 INFO (IPC Server handler 6 on 52615) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38289 is added to blk_1073741825_1001 size 71
[junit4] 2> 2427484 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2427484 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2427484 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2427489 INFO (IPC Server handler 8 on 52615) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:58758 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-165617a1-5bfd-453b-9219-8edf920eacd9:NORMAL:127.0.0.1:38289|RBW], ReplicaUC[[DISK]DS-31a6424c-08ad-4f03-8663-73d81dac5516:NORMAL:127.0.0.1:58758|RBW]]} size 0
[junit4] 2> 2427491 INFO (IPC Server handler 4 on 52615) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:38289 is added to blk_1073741826_1002 size 71
[junit4] 2> 2428071 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2428071 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2428077 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 2428077 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2428077 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 2428090 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7a0b78f[recoverytest_shard1_replica1] main]
[junit4] 2> 2428091 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2428091 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2428092 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2428093 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2428093 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2428093 INFO (searcherExecutor-9716-thread-1-processing-n:127.0.0.1:46673_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest r:core_node1) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@7a0b78f[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2428094 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1552163609534005248
[junit4] 2> 2428105 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 2428107 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@10c338bd[recoverytest_shard1_replica2] main]
[junit4] 2> 2428109 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2428109 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2428109 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 2428109 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1552163609549733888
[junit4] 2> 2428113 INFO (searcherExecutor-9717-thread-1-processing-n:127.0.0.1:48362_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest r:core_node2) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@10c338bd[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2428116 INFO (zkCallback-10872-thread-3-processing-n:127.0.0.1:46673_solr) [n:127.0.0.1:46673_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2428116 INFO (zkCallback-10871-thread-1-processing-n:127.0.0.1:48362_solr) [n:127.0.0.1:48362_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2428611 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2428611 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2428611 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/
[junit4] 2> 2428612 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=https://127.0.0.1:46673/solr START replicas=[https://127.0.0.1:48362/solr/recoverytest_shard1_replica2/] nUpdates=100
[junit4] 2> 2428612 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=https://127.0.0.1:46673/solr DONE. We have no versions. sync failed.
[junit4] 2> 2428626 INFO (qtp634849440-48851) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.c.S.Request [recoverytest_shard1_replica2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2428628 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 2428628 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 2428634 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/ shard1
[junit4] 2> 2428739 INFO (zkCallback-10872-thread-3-processing-n:127.0.0.1:46673_solr) [n:127.0.0.1:46673_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2428739 INFO (zkCallback-10871-thread-1-processing-n:127.0.0.1:48362_solr) [n:127.0.0.1:48362_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2428786 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2428805 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=2462
[junit4] 2> 2428909 INFO (zkCallback-10871-thread-1-processing-n:127.0.0.1:48362_solr) [n:127.0.0.1:48362_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2428910 INFO (zkCallback-10872-thread-3-processing-n:127.0.0.1:46673_solr) [n:127.0.0.1:46673_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2429119 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=2760
[junit4] 2> 2429122 INFO (qtp2127796789-48839) [n:127.0.0.1:46673_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2429227 INFO (zkCallback-10872-thread-2-processing-n:127.0.0.1:46673_solr) [n:127.0.0.1:46673_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2429227 INFO (zkCallback-10871-thread-1-processing-n:127.0.0.1:48362_solr) [n:127.0.0.1:48362_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2430123 INFO (qtp2127796789-48839) [n:127.0.0.1:46673_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3909
[junit4] 2> 2430126 INFO (TEST-HdfsRecoveryZkTest.test-seed#[8B11045FDE87FCC5]) [ ] o.a.s.c.RecoveryZkTest Indexing 700 documents
[junit4] 2> 2430174 INFO (qtp634849440-48848) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-0 (1552163611680440320)]} 0 18
[junit4] 2> 2430175 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1552163611680440320)]} 0 34
[junit4] 2> 2430176 INFO (qtp634849440-48847) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-0 (1552163611673100288)]} 0 11
[junit4] 2> 2430177 INFO (qtp2127796789-48840) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1552163611673100288)]} 0 44
[junit4] 2> 2430182 INFO (qtp634849440-48846) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-1 (1552163611719237632)]} 0 2
[junit4] 2> 2430182 INFO (qtp634849440-48851) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-1 (1552163611719237633)]} 0 2
[junit4] 2> 2430182 INFO (qtp2127796789-48839) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1552163611719237632)]} 0 4
[junit4] 2> 2430183 INFO (qtp2127796789-48836) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1552163611719237633)]} 0 4
[junit4] 2> 2430190 INFO (qtp634849440-48853) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-2 (1552163611727626240)]} 0 1
[junit4] 2> 2430190 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-2 (1552163611728674816)]} 0 1
[junit4] 2> 2430190 INFO (qtp2127796789-48841) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1552163611728674816)]} 0 3
[junit4] 2> 2430190 INFO (qtp2127796789-48837) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1552163611727626240)]} 0 3
[junit4] 2> 2430195 INFO (qtp634849440-48852) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-3 (1552163611733917696)]} 0 0
[junit4] 2> 2430195 INFO (qtp634849440-48848) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-3 (1552163611733917697)]} 0 0
[junit4] 2> 2430195 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1552163611733917697)]} 0 2
[junit4] 2> 2430195 INFO (qtp2127796789-48843) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1552163611733917696)]} 0 2
[junit4] 2> 2430200 INFO (qtp634849440-48846) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-1 (-1552163611739160577)]} 0 0
[junit4] 2> 2430200 INFO (qtp634849440-48847) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-1 (-1552163611739160576)]} 0 1
[junit4] 2> 2430200 INFO (qtp2127796789-48839) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-1 (-1552163611739160577)]} 0 3
[junit4] 2> 2430200 INFO (qtp2127796789-48840) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-1 (-1552163611739160576)]} 0 3
[junit4] 2> 2430205 INFO (qtp634849440-48853) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-4 (1552163611744403457)]} 0 0
[junit4] 2> 2430205 INFO (qtp634849440-48851) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-4 (1552163611744403456)]} 0 1
[junit4] 2> 2430205 INFO (qtp2127796789-48841) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1552163611744403457)]} 0 3
[junit4] 2> 2430206 INFO (qtp2127796789-48836) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1552163611744403456)]} 0 3
[junit4] 2> 2430212 INFO (qtp634849440-48852) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-2 (-1552163611750694913)]} 0 1
[junit4] 2> 2430212 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-2 (-1552163611750694912)]} 0 1
[junit4] 2> 2430212 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-2 (-1552163611750694913)]} 0 3
[junit4] 2> 2430212 INFO (qtp2127796789-48837) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-2 (-1552163611750694912)]} 0 3
[junit4] 2> 2430218 INFO (qtp634849440-48848) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-5 (1552163611758034944)]} 0 1
[junit4] 2> 2430218 INFO (qtp634849440-48846) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-5 (1552163611758034945)]} 0 1
[junit4] 2> 2430218 INFO (qtp2127796789-48839) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1552163611758034945)]} 0 3
[junit4] 2> 2430218 INFO (qtp2127796789-48843) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1552163611758034944)]} 0 3
[junit4] 2> 2430225 INFO (qtp634849440-48847) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-6 (1552163611764326400)]} 0 1
[junit4] 2> 2430225 INFO (qtp634849440-48853) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-6 (1552163611764326401)]} 0 1
[junit4] 2> 2430225 INFO (qtp2127796789-48841) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1552163611764326401)]} 0 3
[junit4] 2> 2430225 INFO (qtp2127796789-48840) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-6 (1552163611764326400)]} 0 4
[junit4] 2> 2430226 INFO (OverseerCollectionConfigSetProcessor-97010225271144454-127.0.0.1:46673_solr-n_0000000000) [n:127.0.0.1:46673_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2430231 INFO (qtp634849440-48851) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-4 (-1552163611770617856)]} 0 1
[junit4] 2> 2430231 INFO (qtp634849440-48852) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-4 (-1552163611771666432)]} 0 1
[junit4] 2> 2430232 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-4 (-1552163611771666432)]} 0 3
[junit4] 2> 2430232 INFO (qtp2127796789-48836) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-4 (-1552163611770617856)]} 0 4
[junit4] 2> 2430236 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-7 (1552163611777957888)]} 0 1
[junit4] 2> 2430237 INFO (qtp634849440-48848) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-7 (1552163611777957889)]} 0 1
[junit4] 2> 2430237 INFO (qtp2127796789-48837) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-7 (1552163611777957888)]} 0 3
[junit4] 2> 2430237 INFO (qtp2127796789-48839) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-7 (1552163611777957889)]} 0 3
[junit4] 2> 2430246 INFO (qtp634849440-48846) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-8 (1552163611787395073)]} 0 0
[junit4] 2> 2430246 INFO (qtp634849440-48847) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-8 (1552163611787395072)]} 0 0
[junit4] 2> 2430246 INFO (qtp2127796789-48843) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-8 (1552163611787395072)]} 0 3
[junit4] 2> 2430246 INFO (qtp2127796789-48841) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-8 (1552163611787395073)]} 0 3
[junit4] 2> 2430255 INFO (qtp634849440-48851) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-8 (-1552163611794735104)]} 0 1
[junit4] 2> 2430256 INFO (qtp634849440-48853) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-8 (-1552163611795783680)]} 0 3
[junit4] 2> 2430256 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-8 (-1552163611795783680)]} 0 5
[junit4] 2> 2430257 INFO (qtp2127796789-48840) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-8 (-1552163611794735104)]} 0 6
[junit4] 2> 2430261 INFO (qtp634849440-48852) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-9 (1552163611803123712)]} 0 1
[junit4] 2> 2430261 INFO (qtp2127796789-48836) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-9 (1552163611803123712)]} 0 3
[junit4] 2> 2430262 INFO (qtp634849440-48850) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-9 (1552163611804172288)]} 0 1
[junit4] 2> 2430262 INFO (qtp2127796789-48837) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-9 (1552163611804172288)]} 0 3
[junit4] 2> 2430267 INFO (qtp634849440-48848) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-10 (1552163611809415168)]} 0 0
[junit4] 2> 2430267 INFO (qtp634849440-48846) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-10 (1552163611809415169)]} 0 1
[junit4] 2> 2430267 INFO (qtp2127796789-48839) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-10 (1552163611809415168)]} 0 3
[junit4] 2> 2430267 INFO (qtp2127796789-48843) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-10 (1552163611809415169)]} 0 3
[junit4] 2> 2430273 INFO (qtp634849440-48851) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-11 (1552163611815706624)]} 0 0
[junit4] 2> 2430273 INFO (qtp634849440-48847) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-11 (1552163611815706625)]} 0 1
[junit4] 2> 2430273 INFO (qtp2127796789-48842) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-11 (1552163611815706625)]} 0 3
[junit4] 2> 2430273 INFO (qtp2127796789-48841) [n:127.0.0.1:46673_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-11 (1552163611815706624)]} 0 3
[junit4] 2> 2430278 INFO (qtp634849440-48853) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=https://127.0.0.1:46673/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-11 (-1552163611821998081)]} 0 0
[junit4] 2> 2430279 INFO (qtp634849440-48852) [n:127.0.0.1:48362_solr c:recoverytest s:shard1 r:core_
[...truncated too long message...]
ckmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 2442458 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33037 33037
[junit4] 2> 2442459 INFO (SUITE-HdfsRecoveryZkTest-seed#[8B11045FDE87FCC5]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: test params are: codec=Lucene62, sim=RandomSimilarity(queryNorm=false,coord=yes): {}, locale=zh-SG, timezone=Australia/NSW
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=2,free=302813440,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [TestPKIAuthenticationPlugin, TestDocumentBuilder, ResponseHeaderTest, TestShardHandlerFactory, TestOrdValues, TestSolrConfigHandlerConcurrent, IndexBasedSpellCheckerTest, TestRandomRequestDistribution, TestManagedSchemaAPI, TestSolrCloudSnapshots, TestConfigReload, TestFieldCacheVsDocValues, HdfsUnloadDistributedZkTest, TestPhraseSuggestions, VersionInfoTest, ShardRoutingTest, TestHashPartitioner, TestCustomDocTransformer, HighlighterTest, CollectionsAPIAsyncDistributedZkTest, TestUpdate, TestComplexPhraseQParserPlugin, TestTrackingShardHandlerFactory, HardAutoCommitTest, RestartWhileUpdatingTest, TestUseDocValuesAsStored2, TestStressReorder, TestComponentsName, TestConfigOverlay, HdfsCollectionsAPIDistributedZkTest, CdcrUpdateLogTest, TestSchemaNameResource, SolrCoreTest, SampleTest, TestLeaderElectionWithEmptyReplica, HttpPartitionTest, TestCryptoKeys, TestDistributedSearch, DistribDocExpirationUpdateProcessorTest, PrimitiveFieldTypeTest, TestSolrIndexConfig, CurrencyFieldXmlFileTest, SpatialHeatmapFacetsTest, TestLeaderInitiatedRecoveryThread, ReplicaListTransformerTest, TestStressRecovery, TestOverriddenPrefixQueryForCustomFieldType, ResponseLogComponentTest, BJQParserTest, TestXmlQParserPlugin, RecoveryZkTest, TestTolerantUpdateProcessorRandomCloud, DisMaxRequestHandlerTest, ConcurrentDeleteAndCreateCollectionTest, TestStressLiveNodes, SolrTestCaseJ4Test, RulesTest, TestNamedUpdateProcessors, DirectSolrConnectionTest, TestPseudoReturnFields, TestExclusionRuleCollectionAccess, SuggestComponentTest, SegmentsInfoRequestHandlerTest, TestJettySolrRunner, TestExactSharedStatsCache, CoreSorterTest, InfoHandlerTest, ZkNodePropsTest, TestInfoStreamLogging, TimeZoneUtilsTest, XsltUpdateRequestHandlerTest, TestInitQParser, FastVectorHighlighterTest, TestScoreJoinQPScore, ShardSplitTest, CdcrRequestHandlerTest, HdfsRestartWhileUpdatingTest, TestMissingGroups, TestPartialUpdateDeduplication, TestRemoteStreaming, NumericFieldsTest, TestTrieFacet, HdfsThreadLeakTest, OutputWriterTest, MergeStrategyTest, TestAnalyzedSuggestions, AlternateDirectoryTest, HdfsDirectoryTest, TestBinaryResponseWriter, DOMUtilTest, TestCloudDeleteByQuery, AssignTest, SolrRequestParserTest, BadComponentTest, TestDocTermOrds, DistributedFacetExistsSmallTest, PeerSyncTest, ZkStateReaderTest, XmlUpdateRequestHandlerTest, DistributedSuggestComponentTest, TestCollectionAPI, AnalyticsQueryTest, TestSerializedLuceneMatchVersion, FileUtilsTest, TermsComponentTest, SynonymTokenizerTest, TestCursorMarkWithoutUniqueKey, ZkCLITest, SOLR749Test, SharedFSAutoReplicaFailoverUtilsTest, DeleteStatusTest, TestReplicaProperties, DistributedFacetPivotLongTailTest, LeaderFailoverAfterPartitionTest, TestFieldTypeCollectionResource, TestCloudPivotFacet, TestSlowCompositeReaderWrapper, TestCharFilters, TestRTGBase, StatelessScriptUpdateProcessorFactoryTest, DistributedQueueTest, SolrPluginUtilsTest, TestGeoJSONResponseWriter, PluginInfoTest, TestJsonFacets, TestFaceting, TestAnalyzeInfixSuggestions, ExplicitHLLTest, TestSolrDynamicMBean, TestBM25SimilarityFactory, TestLRUStatsCache, DirectUpdateHandlerTest, TestDynamicFieldCollectionResource, JSONWriterTest, SimplePostToolTest, PolyFieldTest, SortSpecParsingTest, TestDistribDocBasedVersion, CursorPagingTest, CloudExitableDirectoryReaderTest, TestExpandComponent, TestHighFrequencyDictionaryFactory, TestObjectReleaseTracker, TestStressLucene, TestReplicationHandlerBackup, TestSubQueryTransformer, CloudMLTQParserTest, HighlighterMaxOffsetTest, CheckHdfsIndexTest, PeerSyncWithIndexFingerprintCachingTest, TestSha256AuthenticationProvider, SuggesterFSTTest, TestRandomDVFaceting, TestSweetSpotSimilarityFactory, SolrCLIZkUtilsTest, MoreLikeThisHandlerTest, HLLSerializationTest, ChangedSchemaMergeTest, TestRawResponseWriter, TestSolrCloudWithSecureImpersonation, TestFunctionQuery, RecoveryAfterSoftCommitTest, TestZkChroot, ShardRoutingCustomTest, AutoCommitTest, TestDistributedGrouping, ZkControllerTest, TestJoin, TestReload, DistributedTermsComponentTest, TestCoreContainer, SimpleFacetsTest, TestGroupingSearch, BadIndexSchemaTest, TestSort, TestFiltering, TestIndexSearcher, SoftAutoCommitTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, TestExtendedDismaxParser, DocValuesTest, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, DocValuesMultiTest, TestSurroundQueryParser, StandardRequestHandlerTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, TestCollationField, ReturnFieldsTest, MBeansHandlerTest, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestLFUCache, TestLuceneMatchVersion, EchoParamsTest, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestJmxMonitoredMap, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, ConnectionManagerTest, CustomCollectionTest, DistribJoinFromCollectionTest, DistributedVersionInfoTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnCommitTest, MigrateRouteKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerStatusTest, OverseerTaskQueueTest, RemoteQueryErrorTest, ReplaceNodeTest, RollingRestartTest, SSLMigrationTest, SaslZkACLProviderTest, SolrXmlInZkTest, TestCloudInspectUtil, TestCloudPseudoReturnFields, TestCloudRecovery, TestLocalFSCloudBackupRestore, TestRandomFlRTGCloud, TestRequestStatusCollectionAPI, TestSolrCloudWithDelegationTokens, TestSolrCloudWithKerberosAlt, TriLevelCompositeIdRoutingTest, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=8B11045FDE87FCC5 -Dtests.slow=true -Dtests.locale=zh-SG -Dtests.timezone=Australia/NSW -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:705)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:767)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1006)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:871)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:775)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([8B11045FDE87FCC5]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:260)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [519/658 (2!)] on J0 in 50.43s, 1 test, 1 failure <<< FAILURES!
[...truncated 11501 lines...]
-validate-source-patterns:
[source-patterns] nocommit: solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:765: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:130: Found 1 violations in source files (nocommit).
Total time: 90 minutes 13 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 517 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/517/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithSourceCluster
Error Message:
Document mismatch on target after sync expected:<20000> but was:<10000>
Stack Trace:
java.lang.AssertionError: Document mismatch on target after sync expected:<20000> but was:<10000>
at __randomizedtesting.SeedInfo.seed([8ECD054F4BAB207A:579B548B48CF3330]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.CdcrBootstrapTest.testBootstrapWithSourceCluster(CdcrBootstrapTest.java:232)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12164 lines...]
[junit4] Suite: org.apache.solr.cloud.CdcrBootstrapTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_8ECD054F4BAB207A-001/init-core-data-001
[junit4] 2> 1833621 INFO (SUITE-CdcrBootstrapTest-seed#[8ECD054F4BAB207A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1833623 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBootstrapWithSourceCluster
[junit4] 2> 1833623 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_8ECD054F4BAB207A-001/cdcr-target-001
[junit4] 2> 1833623 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1833623 INFO (Thread-5771) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1833623 INFO (Thread-5771) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1833723 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.ZkTestServer start zk server on port:44817
[junit4] 2> 1833732 INFO (jetty-launcher-3131-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1833734 INFO (jetty-launcher-3131-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@11bf33bd{/solr,null,AVAILABLE}
[junit4] 2> 1833734 INFO (jetty-launcher-3131-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@76e454d0{HTTP/1.1,[http/1.1]}{127.0.0.1:49615}
[junit4] 2> 1833735 INFO (jetty-launcher-3131-thread-1) [ ] o.e.j.s.Server Started @1837751ms
[junit4] 2> 1833735 INFO (jetty-launcher-3131-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49615}
[junit4] 2> 1833735 ERROR (jetty-launcher-3131-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1833735 INFO (jetty-launcher-3131-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 1833735 INFO (jetty-launcher-3131-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1833735 INFO (jetty-launcher-3131-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1833735 INFO (jetty-launcher-3131-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-27T04:42:44.591Z
[junit4] 2> 1833738 INFO (jetty-launcher-3131-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1833745 INFO (jetty-launcher-3131-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1833745 INFO (jetty-launcher-3131-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44817/solr
[junit4] 2> 1833782 INFO (jetty-launcher-3131-thread-1) [n:127.0.0.1:49615_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49615_solr
[junit4] 2> 1833783 INFO (jetty-launcher-3131-thread-1) [n:127.0.0.1:49615_solr ] o.a.s.c.Overseer Overseer (id=97007813557026819-127.0.0.1:49615_solr-n_0000000000) starting
[junit4] 2> 1833794 INFO (jetty-launcher-3131-thread-1) [n:127.0.0.1:49615_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49615_solr
[junit4] 2> 1833797 INFO (OverseerStateUpdate-97007813557026819-127.0.0.1:49615_solr-n_0000000000) [n:127.0.0.1:49615_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1833878 INFO (jetty-launcher-3131-thread-1) [n:127.0.0.1:49615_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_8ECD054F4BAB207A-001/cdcr-target-001/node1/.
[junit4] 1> Target zkHost = 127.0.0.1:44817/solr
[junit4] 2> 1833886 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_8ECD054F4BAB207A-001/cdcr-source-001
[junit4] 2> 1833887 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1833887 INFO (Thread-5781) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1833887 INFO (Thread-5781) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1833890 WARN (Thread-5781) [ ] o.a.z.j.MBeanRegistry Failed to register MBean StandaloneServer_port-1
[junit4] 2> 1833890 WARN (Thread-5781) [ ] o.a.z.s.ZooKeeperServer Failed to register with JMX
[junit4] 2> javax.management.InstanceAlreadyExistsException: org.apache.ZooKeeperService:name0=StandaloneServer_port-1
[junit4] 2> at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900)
[junit4] 2> at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324)
[junit4] 2> at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522)
[junit4] 2> at org.apache.zookeeper.jmx.MBeanRegistry.register(MBeanRegistry.java:96)
[junit4] 2> at org.apache.zookeeper.server.ZooKeeperServer.registerJMX(ZooKeeperServer.java:377)
[junit4] 2> at org.apache.zookeeper.server.ZooKeeperServer.startup(ZooKeeperServer.java:410)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.startup(NIOServerCnxnFactory.java:123)
[junit4] 2> at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:308)
[junit4] 2> at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
[junit4] 2> 1833988 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.ZkTestServer start zk server on port:44332
[junit4] 2> 1833997 INFO (jetty-launcher-3142-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1833999 INFO (jetty-launcher-3142-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@dd0ffe1{/solr,null,AVAILABLE}
[junit4] 2> 1833999 INFO (jetty-launcher-3142-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5a259fee{HTTP/1.1,[http/1.1]}{127.0.0.1:50243}
[junit4] 2> 1833999 INFO (jetty-launcher-3142-thread-1) [ ] o.e.j.s.Server Started @1838016ms
[junit4] 2> 1833999 INFO (jetty-launcher-3142-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50243}
[junit4] 2> 1834000 ERROR (jetty-launcher-3142-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1834000 INFO (jetty-launcher-3142-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 1834000 INFO (jetty-launcher-3142-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1834000 INFO (jetty-launcher-3142-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1834000 INFO (jetty-launcher-3142-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-27T04:42:44.856Z
[junit4] 2> 1834003 INFO (jetty-launcher-3142-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1834008 INFO (jetty-launcher-3142-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1834009 INFO (jetty-launcher-3142-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44332/solr
[junit4] 2> 1834045 INFO (jetty-launcher-3142-thread-1) [n:127.0.0.1:50243_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50243_solr
[junit4] 2> 1834047 INFO (jetty-launcher-3142-thread-1) [n:127.0.0.1:50243_solr ] o.a.s.c.Overseer Overseer (id=97007813574262787-127.0.0.1:50243_solr-n_0000000000) starting
[junit4] 2> 1834062 INFO (jetty-launcher-3142-thread-1) [n:127.0.0.1:50243_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50243_solr
[junit4] 2> 1834064 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:50243_solr) [n:127.0.0.1:50243_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1834302 INFO (jetty-launcher-3142-thread-1) [n:127.0.0.1:50243_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_8ECD054F4BAB207A-001/cdcr-source-001/node1/.
[junit4] 2> 1834325 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1834328 INFO (TEST-CdcrBootstrapTest.testBootstrapWithSourceCluster-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44332/solr ready
[junit4] 2> 1834330 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1834333 INFO (OverseerThreadFactory-8364-thread-1-processing-n:127.0.0.1:50243_solr) [n:127.0.0.1:50243_solr ] o.a.s.c.CreateCollectionCmd Create collection cdcr-source
[junit4] 2> 1834441 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&name=cdcr-source_shard1_replica1&action=CREATE&numShards=1&collection=cdcr-source&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2
[junit4] 2> 1834551 INFO (zkCallback-3148-thread-2-processing-n:127.0.0.1:50243_solr) [n:127.0.0.1:50243_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1835455 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 1835481 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 x:cdcr-source_shard1_replica1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 1835483 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 x:cdcr-source_shard1_replica1] o.a.s.s.IndexSchema [cdcr-source_shard1_replica1] Schema name=minimal
[junit4] 2> 1835485 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 x:cdcr-source_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 1835486 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 x:cdcr-source_shard1_replica1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/cdcr-source/managed-schema
[junit4] 2> 1835492 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 x:cdcr-source_shard1_replica1] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/cdcr-source/schema.xml to /configs/cdcr-source/schema.xml.bak
[junit4] 2> 1835493 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 x:cdcr-source_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-source_shard1_replica1' using configuration from collection cdcr-source
[junit4] 2> 1835493 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [[cdcr-source_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_8ECD054F4BAB207A-001/cdcr-source-001/node1/cdcr-source_shard1_replica1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_8ECD054F4BAB207A-001/cdcr-source-001/node1/./cdcr-source_shard1_replica1/data/]
[junit4] 2> 1835534 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 1835534 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1835535 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1835535 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1835536 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@64fa6088[cdcr-source_shard1_replica1] main]
[junit4] 2> 1835538 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-source
[junit4] 2> 1835540 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-source
[junit4] 2> 1835540 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-source/managed-schema
[junit4] 2> 1835541 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1835546 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-source/cdcr/state/buffer
[junit4] 2> 1835548 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-source/cdcr/state/process
[junit4] 2> 1835556 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@64fa6088[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1835556 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1552125018937229312
[junit4] 2> 1835571 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1835571 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1835571 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:50243/solr/cdcr-source_shard1_replica1/
[junit4] 2> 1835571 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1835571 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:50243/solr/cdcr-source_shard1_replica1/ has no replicas
[junit4] 2> 1835579 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:50243_solr) [n:127.0.0.1:50243_solr ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-source:shard1
[junit4] 2> 1835580 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:50243/solr/cdcr-source_shard1_replica1/ shard1
[junit4] 2> 1835691 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:50243_solr) [n:127.0.0.1:50243_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1835735 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1835738 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-source&newCollection=true&name=cdcr-source_shard1_replica1&action=CREATE&numShards=1&collection=cdcr-source&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=1297
[junit4] 2> 1835742 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1835847 INFO (zkCallback-3148-thread-1-processing-n:127.0.0.1:50243_solr) [n:127.0.0.1:50243_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-source/state.json] for collection [cdcr-source] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1836339 INFO (OverseerCollectionConfigSetProcessor-97007813574262787-127.0.0.1:50243_solr-n_0000000000) [n:127.0.0.1:50243_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1836743 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=cdcr-source&name=cdcr-source&action=CREATE&numShards=1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2414
[junit4] 1> Adding 100 docs with commit=true, numDocs=100
[junit4] 2> 1836755 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 1836757 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1836757 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1836778 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@66096a2[cdcr-source_shard1_replica1] main]
[junit4] 2> 1836779 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@66096a2[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100)))}
[junit4] 2> 1836779 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1836779 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=21
[junit4] 1> Adding 100 docs with commit=true, numDocs=200
[junit4] 2> 1836787 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 1836790 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1836790 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1836812 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2a4b2f3[cdcr-source_shard1_replica1] main]
[junit4] 2> 1836812 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@2a4b2f3[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100)))}
[junit4] 2> 1836812 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1836813 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=22
[junit4] 1> Adding 100 docs with commit=true, numDocs=300
[junit4] 2> 1836819 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 1836821 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1836821 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1836842 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@26058c51[cdcr-source_shard1_replica1] main]
[junit4] 2> 1836843 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1836843 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@26058c51[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100) Uninverting(_2(6.4.0):C100)))}
[junit4] 2> 1836843 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=21
[junit4] 1> Adding 100 docs with commit=true, numDocs=400
[junit4] 2> 1836851 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 1836854 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1836854 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1836880 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@ca73bcc[cdcr-source_shard1_replica1] main]
[junit4] 2> 1836880 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@ca73bcc[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100) Uninverting(_2(6.4.0):C100) Uninverting(_3(6.4.0):C100)))}
[junit4] 2> 1836880 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1836881 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=27
[junit4] 1> Adding 100 docs with commit=true, numDocs=500
[junit4] 2> 1836888 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 1836896 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1836896 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1836923 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@78ffab88[cdcr-source_shard1_replica1] main]
[junit4] 2> 1836923 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@78ffab88[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100) Uninverting(_2(6.4.0):C100) Uninverting(_3(6.4.0):C100) Uninverting(_4(6.4.0):C100)))}
[junit4] 2> 1836923 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1836923 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=27
[junit4] 1> Adding 100 docs with commit=true, numDocs=600
[junit4] 2> 1836932 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 1836934 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1836934 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1836961 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1e80d1ed[cdcr-source_shard1_replica1] main]
[junit4] 2> 1836962 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1836962 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@1e80d1ed[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100) Uninverting(_2(6.4.0):C100) Uninverting(_3(6.4.0):C100) Uninverting(_4(6.4.0):C100) Uninverting(_5(6.4.0):C100)))}
[junit4] 2> 1836962 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=28
[junit4] 1> Adding 100 docs with commit=true, numDocs=700
[junit4] 2> 1836968 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1836980 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1836980 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837008 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@36e1e70c[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837009 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837009 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@36e1e70c[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100) Uninverting(_2(6.4.0):C100) Uninverting(_3(6.4.0):C100) Uninverting(_4(6.4.0):C100) Uninverting(_5(6.4.0):C100) Uninverting(_6(6.4.0):C100)))}
[junit4] 2> 1837009 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=29
[junit4] 1> Adding 100 docs with commit=true, numDocs=800
[junit4] 2> 1837018 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1837021 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837021 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837052 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1faa6974[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837052 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837052 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@1faa6974[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100) Uninverting(_2(6.4.0):C100) Uninverting(_3(6.4.0):C100) Uninverting(_4(6.4.0):C100) Uninverting(_5(6.4.0):C100) Uninverting(_6(6.4.0):C100) Uninverting(_7(6.4.0):C100)))}
[junit4] 2> 1837053 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=31
[junit4] 1> Adding 100 docs with commit=true, numDocs=900
[junit4] 2> 1837060 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837063 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837063 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837092 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@52e33ddb[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837093 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837093 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@52e33ddb[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100) Uninverting(_2(6.4.0):C100) Uninverting(_3(6.4.0):C100) Uninverting(_4(6.4.0):C100) Uninverting(_5(6.4.0):C100) Uninverting(_6(6.4.0):C100) Uninverting(_7(6.4.0):C100) Uninverting(_8(6.4.0):C100)))}
[junit4] 2> 1837093 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=30
[junit4] 1> Adding 100 docs with commit=true, numDocs=1000
[junit4] 2> 1837100 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1837102 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837102 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837137 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d7dc570[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837138 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837138 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@6d7dc570[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(6.4.0):C100) Uninverting(_1(6.4.0):C100) Uninverting(_2(6.4.0):C100) Uninverting(_3(6.4.0):C100) Uninverting(_4(6.4.0):C100) Uninverting(_5(6.4.0):C100) Uninverting(_6(6.4.0):C100) Uninverting(_7(6.4.0):C100) Uninverting(_8(6.4.0):C100) Uninverting(_9(6.4.0):C100)))}
[junit4] 2> 1837138 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=36
[junit4] 1> Adding 100 docs with commit=true, numDocs=1100
[junit4] 2> 1837146 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1837151 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837151 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837183 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5ce0e331[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837184 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@5ce0e331[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100)))}
[junit4] 2> 1837184 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837209 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=59
[junit4] 1> Adding 100 docs with commit=true, numDocs=1200
[junit4] 2> 1837214 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837216 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837216 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837240 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e28e49d[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837241 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837241 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@5e28e49d[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100) Uninverting(_c(6.4.0):C100)))}
[junit4] 2> 1837241 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=24
[junit4] 1> Adding 100 docs with commit=true, numDocs=1300
[junit4] 2> 1837245 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837248 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837248 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837275 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3377924a[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837276 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@3377924a[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100) Uninverting(_c(6.4.0):C100) Uninverting(_d(6.4.0):C100)))}
[junit4] 2> 1837276 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837276 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=28
[junit4] 1> Adding 100 docs with commit=true, numDocs=1400
[junit4] 2> 1837281 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837285 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837286 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837316 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@14d21765[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837316 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@14d21765[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100) Uninverting(_c(6.4.0):C100) Uninverting(_d(6.4.0):C100) Uninverting(_e(6.4.0):C100)))}
[junit4] 2> 1837316 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837317 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=31
[junit4] 1> Adding 100 docs with commit=true, numDocs=1500
[junit4] 2> 1837324 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1837327 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837327 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837355 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d9db59b[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837355 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837355 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@1d9db59b[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100) Uninverting(_c(6.4.0):C100) Uninverting(_d(6.4.0):C100) Uninverting(_e(6.4.0):C100) Uninverting(_f(6.4.0):C100)))}
[junit4] 2> 1837356 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=28
[junit4] 1> Adding 100 docs with commit=true, numDocs=1600
[junit4] 2> 1837362 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837364 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837364 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837397 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@436ed90f[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837398 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837398 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@436ed90f[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100) Uninverting(_c(6.4.0):C100) Uninverting(_d(6.4.0):C100) Uninverting(_e(6.4.0):C100) Uninverting(_f(6.4.0):C100) Uninverting(_g(6.4.0):C100)))}
[junit4] 2> 1837399 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=35
[junit4] 1> Adding 100 docs with commit=true, numDocs=1700
[junit4] 2> 1837407 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1837410 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837410 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837460 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d4de339[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837460 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837461 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@1d4de339[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100) Uninverting(_c(6.4.0):C100) Uninverting(_d(6.4.0):C100) Uninverting(_e(6.4.0):C100) Uninverting(_f(6.4.0):C100) Uninverting(_g(6.4.0):C100) Uninverting(_h(6.4.0):C100)))}
[junit4] 2> 1837465 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=55
[junit4] 1> Adding 100 docs with commit=true, numDocs=1800
[junit4] 2> 1837473 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 1837475 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837475 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837523 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@d68123d[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837523 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837523 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@d68123d[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100) Uninverting(_c(6.4.0):C100) Uninverting(_d(6.4.0):C100) Uninverting(_e(6.4.0):C100) Uninverting(_f(6.4.0):C100) Uninverting(_g(6.4.0):C100) Uninverting(_h(6.4.0):C100) Uninverting(_i(6.4.0):C100)))}
[junit4] 2> 1837524 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=48
[junit4] 1> Adding 100 docs with commit=true, numDocs=1900
[junit4] 2> 1837530 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837532 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837532 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837577 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@ab8bee9[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837578 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837578 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@ab8bee9[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_a(6.4.0):C1000) Uninverting(_b(6.4.0):C100) Uninverting(_c(6.4.0):C100) Uninverting(_d(6.4.0):C100) Uninverting(_e(6.4.0):C100) Uninverting(_f(6.4.0):C100) Uninverting(_g(6.4.0):C100) Uninverting(_h(6.4.0):C100) Uninverting(_i(6.4.0):C100) Uninverting(_j(6.4.0):C100)))}
[junit4] 2> 1837578 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=46
[junit4] 1> Adding 100 docs with commit=true, numDocs=2000
[junit4] 2> 1837588 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=6
[junit4] 2> 1837591 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837591 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837628 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@47fc386e[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837628 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@47fc386e[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100)))}
[junit4] 2> 1837628 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837655 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=63
[junit4] 1> Adding 100 docs with commit=true, numDocs=2100
[junit4] 2> 1837660 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837662 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837662 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837686 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7fda187f[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837686 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@7fda187f[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100) Uninverting(_m(6.4.0):C100)))}
[junit4] 2> 1837686 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837687 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=24
[junit4] 1> Adding 100 docs with commit=true, numDocs=2200
[junit4] 2> 1837693 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1837695 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837695 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837722 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@41b9e969[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837722 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837722 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@41b9e969[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100) Uninverting(_m(6.4.0):C100) Uninverting(_n(6.4.0):C100)))}
[junit4] 2> 1837722 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=27
[junit4] 1> Adding 100 docs with commit=true, numDocs=2300
[junit4] 2> 1837729 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837731 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837731 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837756 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3b3465a7[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837757 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837757 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@3b3465a7[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100) Uninverting(_m(6.4.0):C100) Uninverting(_n(6.4.0):C100) Uninverting(_o(6.4.0):C100)))}
[junit4] 2> 1837757 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=26
[junit4] 1> Adding 100 docs with commit=true, numDocs=2400
[junit4] 2> 1837766 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837769 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837769 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837797 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3b79aac4[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837797 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837797 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@3b79aac4[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100) Uninverting(_m(6.4.0):C100) Uninverting(_n(6.4.0):C100) Uninverting(_o(6.4.0):C100) Uninverting(_p(6.4.0):C100)))}
[junit4] 2> 1837797 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=28
[junit4] 1> Adding 100 docs with commit=true, numDocs=2500
[junit4] 2> 1837802 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837804 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837804 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837834 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d694d39[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837835 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@1d694d39[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100) Uninverting(_m(6.4.0):C100) Uninverting(_n(6.4.0):C100) Uninverting(_o(6.4.0):C100) Uninverting(_p(6.4.0):C100) Uninverting(_q(6.4.0):C100)))}
[junit4] 2> 1837835 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837835 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=31
[junit4] 1> Adding 100 docs with commit=true, numDocs=2600
[junit4] 2> 1837839 INFO (qtp195912523-18631) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1837846 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837846 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837878 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1e2b367b[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837879 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837879 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@1e2b367b[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100) Uninverting(_m(6.4.0):C100) Uninverting(_n(6.4.0):C100) Uninverting(_o(6.4.0):C100) Uninverting(_p(6.4.0):C100) Uninverting(_q(6.4.0):C100) Uninverting(_r(6.4.0):C100)))}
[junit4] 2> 1837880 INFO (qtp195912523-18625) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=34
[junit4] 1> Adding 100 docs with commit=true, numDocs=2700
[junit4] 2> 1837886 INFO (qtp195912523-18632) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1837888 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837888 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837920 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7f07cf95[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837920 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@7f07cf95[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100) Uninverting(_m(6.4.0):C100) Uninverting(_n(6.4.0):C100) Uninverting(_o(6.4.0):C100) Uninverting(_p(6.4.0):C100) Uninverting(_q(6.4.0):C100) Uninverting(_r(6.4.0):C100) Uninverting(_s(6.4.0):C100)))}
[junit4] 2> 1837920 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1837920 INFO (qtp195912523-18628) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=32
[junit4] 1> Adding 100 docs with commit=true, numDocs=2800
[junit4] 2> 1837932 INFO (qtp195912523-18629) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 1837934 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1837934 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1837997 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1dc160cb[cdcr-source_shard1_replica1] main]
[junit4] 2> 1837998 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1838000 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCore [cdcr-source_shard1_replica1] Registered new searcher Searcher@1dc160cb[cdcr-source_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_k(6.4.0):C1900) Uninverting(_l(6.4.0):C100) Uninverting(_m(6.4.0):C100) Uninverting(_n(6.4.0):C100) Uninverting(_o(6.4.0):C100) Uninverting(_p(6.4.0):C100) Uninverting(_q(6.4.0):C100) Uninverting(_r(6.4.0):C100) Uninverting(_s(6.4.0):C100) Uninverting(_t(6.4.0):C100)))}
[junit4] 2> 1838000 INFO (qtp195912523-18627) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=65
[junit4] 1> Adding 100 docs with commit=true, numDocs=2900
[junit4] 2> 1838005 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.S.Request [cdcr-source_shard1_replica1] webapp=/solr path=/update params={_stateVer_=cdcr-source:3&wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 1838008 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1838008 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@113f07e9
[junit4] 2> 1838055 INFO (qtp195912523-18630) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7b81e6a1[cdcr-source_shard1_replica1] main]
[junit4] 2> 1838055 INFO (searcherExecutor-8367-thread-1-processing-n:127.0.0.1:50243_solr x:cdcr-source_shard1_replica1 s:shard1 c:cdcr-source r:core_node1) [n:127.0.0.1:50243_solr c:cdcr-source s:shard1 r:core_node1 x:cdcr-source_shard1_replica1] o.a.s.c.SolrCo
[...truncated too long message...]
nodes
[junit4] 2>
[junit4] 2> 2020093 INFO (jetty-launcher-3196-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@23773c1e{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2020094 INFO (jetty-launcher-3196-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2066041936
[junit4] 2> 2020097 INFO (coreCloseExecutor-8543-thread-1) [n:127.0.0.1:63903_solr c:cdcr-target s:shard1 r:core_node1 x:cdcr-target_shard1_replica1] o.a.s.c.SolrCore [cdcr-target_shard1_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@70f8b86e
[junit4] 2> 2020097 INFO (coreCloseExecutor-8543-thread-1) [n:127.0.0.1:63903_solr c:cdcr-target s:shard1 r:core_node1 x:cdcr-target_shard1_replica1] o.a.s.h.CdcrRequestHandler Solr core is being closed - shutting down CDCR handler @ cdcr-target:shard1
[junit4] 2> 2020105 INFO (jetty-launcher-3196-thread-1) [ ] o.a.s.c.Overseer Overseer (id=97007824525393923-127.0.0.1:63903_solr-n_0000000000) closing
[junit4] 2> 2020105 INFO (OverseerStateUpdate-97007824525393923-127.0.0.1:63903_solr-n_0000000000) [n:127.0.0.1:63903_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:63903_solr
[junit4] 2> 2020106 INFO (zkCallback-3202-thread-5-processing-n:127.0.0.1:63903_solr) [n:127.0.0.1:63903_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-target/state.json] for collection [cdcr-target] has occurred - updating... (live nodes size: [1])
[junit4] 2> 2021610 WARN (zkCallback-3202-thread-5-processing-n:127.0.0.1:63903_solr) [n:127.0.0.1:63903_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /collections/cdcr-target/state.json]
[junit4] 2> 2021610 INFO (jetty-launcher-3196-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3d969870{/solr,null,UNAVAILABLE}
[junit4] 2> 2021615 INFO (TEST-CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap-seed#[8ECD054F4BAB207A]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43498 43498
[junit4] 2> 2021639 INFO (Thread-5866) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:43498 43498
[junit4] 2> 2021640 WARN (Thread-5866) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/configs/cdcr-target/managed-schema
[junit4] 2> 2 /solr/collections/cdcr-target/leaders/shard1/leader
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/collections/cdcr-target/cdcr/state/process
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 2 /solr/collections/cdcr-target/cdcr/state/buffer
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 17 /solr/overseer/collection-queue-work
[junit4] 2> 5 /solr/overseer/queue
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2>
[junit4] 2> 2021641 INFO (TEST-CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap-seed#[8ECD054F4BAB207A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testConvertClusterToCdcrAndBootstrap
[junit4] 2> 2021642 INFO (SUITE-CdcrBootstrapTest-seed#[8ECD054F4BAB207A]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrBootstrapTest_8ECD054F4BAB207A-001
[junit4] 2> Nov 27, 2016 4:45:52 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene62): {_version_=PostingsFormat(name=LuceneFixedGap), xyz=PostingsFormat(name=MockRandom), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{}, maxPointsInLeafNode=900, maxMBSortInHeap=5.949910311828333, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=en-AU, timezone=Libya
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=304128320,total=484970496
[junit4] 2> NOTE: All tests run in this JVM: [RuleEngineTest, RAMDirectoryFactoryTest, HdfsRecoveryZkTest, ClusterStateTest, TestCloudSchemaless, TestMaxScoreQueryParser, TestSchemaResource, AutoCommitTest, DateRangeFieldTest, BufferStoreTest, TestNumericTerms32, TestMacroExpander, ResourceLoaderTest, OverseerStatusTest, ShowFileRequestHandlerTest, TestSearchPerf, TestSolrConfigHandlerCloud, TestSolrCoreSnapshots, SimpleCollectionCreateDeleteTest, CoreAdminCreateDiscoverTest, JavabinLoaderTest, CollectionReloadTest, TestStressCloudBlindAtomicUpdates, TestConfigSetsAPIZkFailure, MinimalSchemaTest, SecurityConfHandlerTest, TestSuggestSpellingConverter, SpellCheckComponentTest, HighlighterConfigTest, TestCloudInspectUtil, TestCollapseQParserPlugin, TestBlobHandler, SuggestComponentContextFilterQueryTest, TestHdfsBackupRestoreCore, TestCustomSort, QueryParsingTest, SignatureUpdateProcessorFactoryTest, BigEndianAscendingWordDeserializerTest, TestScoreJoinQPNoScore, DistributedExpandComponentTest, HdfsChaosMonkeySafeLeaderTest, TestUnifiedSolrHighlighter, TestInitParams, TestSolrCoreProperties, BitVectorTest, TestBinaryResponseWriter, SortByFunctionTest, TestSubQueryTransformerCrossCore, TestUninvertingReader, TestDynamicFieldResource, TestRealTimeGet, DirectoryFactoryTest, TestLMJelinekMercerSimilarityFactory, BlockJoinFacetDistribTest, OpenCloseCoreStressTest, TestUseDocValuesAsStored, CurrencyFieldOpenExchangeTest, TestDownShardTolerantSearch, TestManagedSynonymFilterFactory, TestFieldCache, TestMiniSolrCloudCluster, LeaderInitiatedRecoveryOnShardRestartTest, DistanceFunctionTest, HdfsNNFailoverTest, HdfsTlogReplayBufferedWhileIndexingTest, TestUniqueKeyFieldResource, TestSolrXml, CdcrReplicationHandlerTest, SliceStateTest, ReturnFieldsTest, TestSolrCloudWithKerberosAlt, SolrCloudExampleTest, DeleteShardTest, TestRecovery, TestLeaderElectionZkExpiry, TestSimpleTrackingShardHandler, BasicDistributedZk2Test, CacheHeaderTest, BlockJoinFacetSimpleTest, SolrIndexSplitterTest, TestSizeLimitedDistributedMap, ShufflingReplicaListTransformerTest, CoreMergeIndexesAdminHandlerTest, PreAnalyzedFieldTest, CSVRequestHandlerTest, BasicFunctionalityTest, UpdateRequestProcessorFactoryTest, DistributedTermsComponentTest, IndexSchemaTest, TestHashQParserPlugin, SolrCoreCheckLockOnStartupTest, TestManagedSchemaThreadSafety, DistributedFacetPivotLargeTest, StatsReloadRaceTest, TestHdfsUpdateLog, TestPHPSerializedResponseWriter, CreateCollectionCleanupTest, TestDeleteCollectionOnDownNodes, TestRestManager, TestBadConfig, TestAuthenticationFramework, SolrCmdDistributorTest, BadIndexSchemaTest, TestJsonFacetRefinement, RollingRestartTest, TlogReplayBufferedWhileIndexingTest, JSONWriterTest, DeleteInactiveReplicaTest, ConvertedLegacyTest, TestSQLHandler, RequestHandlersTest, TestCustomStream, FieldMutatingUpdateProcessorTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestUtils, TestRebalanceLeaders, TestLuceneMatchVersion, TestFastWriter, NoCacheHeaderTest, MBeansHandlerTest, ZkStateWriterTest, HdfsBasicDistributedZkTest, HLLUtilTest, FieldAnalysisRequestHandlerTest, ConfigSetsAPITest, TestQuerySenderNoQuery, AnalysisErrorHandlingTest, TestManagedSchema, SuggesterWFSTTest, TestReqParamsAPI, DistributedFacetPivotWhiteBoxTest, TestMinMaxOnMultiValuedField, CleanupOldIndexTest, TestReversedWildcardFilterFactory, TestRandomFlRTGCloud, WordBreakSolrSpellCheckerTest, TestSmileRequest, EchoParamsTest, ExternalFileFieldSortTest, TestDistributedStatsComponentCardinality, TestRequestForwarding, URLClassifyProcessorTest, SharedFSAutoReplicaFailoverTest, TestSchemaVersionResource, DistanceUnitsTest, TestFreeTextSuggestions, RankQueryTest, TestSystemIdResolver, HdfsDirectoryFactoryTest, TestFieldCacheWithThreads, SuggesterTSTTest, DefaultValueUpdateProcessorTest, VMParamsZkACLAndCredentialsProvidersTest, TestAddFieldRealTimeGet, CachingDirectoryFactoryTest, TestConfigSetImmutable, TestStressUserVersions, TestRangeQuery, HdfsWriteToMultipleCollectionsTest, TestDistributedGrouping, TestTestInjection, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, OverseerTest, LeaderElectionIntegrationTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestRandomFaceting, TestZkChroot, TestRandomDVFaceting, ShardRoutingCustomTest, TestFaceting, TermVectorComponentDistributedTest, ZkControllerTest, TestStressReorder, TestJoin, HardAutoCommitTest, StatsComponentTest, HighlighterTest, DistributedQueryElevationComponentTest, TestCoreDiscovery, TestExtendedDismaxParser, SpellCheckCollatorTest, CoreAdminHandlerTest, DocValuesTest, SolrInfoMBeanTest, LukeRequestHandlerTest, TestWriterPerf, FileBasedSpellCheckerTest, TestValueSourceCache, TestIndexingPerformance, MoreLikeThisHandlerTest, FastVectorHighlighterTest, LoggingHandlerTest, JSONWriterTest, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, SearchHandlerTest, TestBinaryResponseWriter, TestLFUCache, TestLMDirichletSimilarityFactory, TestJmxMonitoredMap, PluginInfoTest, DateMathParserTest, DateFieldTest, DOMUtilTest, ZkNodePropsTest, FileUtilsTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestTolerantSearch, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AsyncCallRequestStatusResponseTest, CdcrBootstrapTest]
[junit4] Completed [479/658 (1!)] on J1 in 188.54s, 3 tests, 1 failure <<< FAILURES!
[...truncated 61383 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 516 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/516/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43) at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130) at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137) at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94) at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102) at sun.reflect.GeneratedConstructorAccessor165.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:705) at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:767) at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1006) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:871) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:775) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842) at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
at sun.reflect.GeneratedConstructorAccessor165.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:705)
at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:767)
at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1006)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:871)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:775)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([54BA97A2AEE84936]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:260)
at sun.reflect.GeneratedMethodAccessor55.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12400 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_54BA97A2AEE84936-001/init-core-data-001
[junit4] 2> 3170088 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 3170089 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_54BA97A2AEE84936-001/tempDir-001
[junit4] 2> 3170089 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3170089 INFO (Thread-4287) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3170089 INFO (Thread-4287) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3170189 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:65350
[junit4] 2> 3170203 INFO (jetty-launcher-9365-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3170204 INFO (jetty-launcher-9365-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@18b104b9{/solr,null,AVAILABLE}
[junit4] 2> 3170205 INFO (jetty-launcher-9365-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 3170205 INFO (jetty-launcher-9365-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@24b4df37{HTTP/1.1,[http/1.1]}{127.0.0.1:55454}
[junit4] 2> 3170205 INFO (jetty-launcher-9365-thread-1) [ ] o.e.j.s.Server Started @3174582ms
[junit4] 2> 3170205 INFO (jetty-launcher-9365-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55454}
[junit4] 2> 3170207 ERROR (jetty-launcher-9365-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3170207 INFO (jetty-launcher-9365-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 3170207 INFO (jetty-launcher-9365-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3170207 INFO (jetty-launcher-9365-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3170207 INFO (jetty-launcher-9365-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-26T17:51:39.076Z
[junit4] 2> 3170208 INFO (jetty-launcher-9365-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6db9e81b{/solr,null,AVAILABLE}
[junit4] 2> 3170208 INFO (jetty-launcher-9365-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1f1e015{HTTP/1.1,[http/1.1]}{127.0.0.1:48041}
[junit4] 2> 3170208 INFO (jetty-launcher-9365-thread-2) [ ] o.e.j.s.Server Started @3174586ms
[junit4] 2> 3170208 INFO (jetty-launcher-9365-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48041}
[junit4] 2> 3170209 ERROR (jetty-launcher-9365-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3170209 INFO (jetty-launcher-9365-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 3170209 INFO (jetty-launcher-9365-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3170209 INFO (jetty-launcher-9365-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3170209 INFO (jetty-launcher-9365-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-26T17:51:39.078Z
[junit4] 2> 3170217 INFO (jetty-launcher-9365-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3170220 INFO (jetty-launcher-9365-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3170230 INFO (jetty-launcher-9365-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3170230 INFO (jetty-launcher-9365-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 3170231 INFO (jetty-launcher-9365-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:65350/solr
[junit4] 2> 3170232 INFO (jetty-launcher-9365-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:65350/solr
[junit4] 2> 3170295 INFO (jetty-launcher-9365-thread-1) [n:127.0.0.1:55454_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:55454_solr
[junit4] 2> 3170297 INFO (jetty-launcher-9365-thread-2) [n:127.0.0.1:48041_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48041_solr
[junit4] 2> 3170304 INFO (jetty-launcher-9365-thread-1) [n:127.0.0.1:55454_solr ] o.a.s.c.Overseer Overseer (id=97005253359042566-127.0.0.1:55454_solr-n_0000000000) starting
[junit4] 2> 3170305 INFO (zkCallback-9376-thread-1-processing-n:127.0.0.1:55454_solr) [n:127.0.0.1:55454_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3170321 INFO (zkCallback-9375-thread-1-processing-n:127.0.0.1:48041_solr) [n:127.0.0.1:48041_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3170373 INFO (jetty-launcher-9365-thread-2) [n:127.0.0.1:48041_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_54BA97A2AEE84936-001/tempDir-001/node2/.
[junit4] 2> 3170375 INFO (jetty-launcher-9365-thread-1) [n:127.0.0.1:55454_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55454_solr
[junit4] 2> 3170385 INFO (zkCallback-9376-thread-1-processing-n:127.0.0.1:55454_solr) [n:127.0.0.1:55454_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3170385 INFO (zkCallback-9375-thread-1-processing-n:127.0.0.1:48041_solr) [n:127.0.0.1:48041_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3170476 INFO (jetty-launcher-9365-thread-1) [n:127.0.0.1:55454_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_54BA97A2AEE84936-001/tempDir-001/node1/.
[junit4] 2> 3170487 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3170489 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:65350/solr ready
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 3170547 WARN (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
[junit4] 2> 3170562 WARN (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3170565 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3170589 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_52504_hdfs____.dnb5te/webapp
[junit4] 2> 3171148 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:52504
[junit4] 2> 3171320 WARN (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3171323 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3171340 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_59394_datanode____ap7781/webapp
[junit4] 2> 3171875 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:59394
[junit4] 2> 3171985 WARN (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 3171989 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log jetty-6.1.26
[junit4] 2> 3172019 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_38184_datanode____3wp9ov/webapp
[junit4] 2> 3172079 INFO (IPC Server handler 3 on 39614) [ ] BlockStateChange BLOCK* processReport: from storage DS-c0252d28-0f8a-4f04-8f1c-8c85cb53463e node DatanodeRegistration(127.0.0.1:37864, datanodeUuid=74100111-f3ba-4d97-bacc-ce9077eebf2a, infoPort=42399, infoSecurePort=0, ipcPort=47702, storageInfo=lv=-56;cid=testClusterID;nsid=661839495;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3172079 INFO (IPC Server handler 3 on 39614) [ ] BlockStateChange BLOCK* processReport: from storage DS-03be733f-f43a-4900-af2a-e72d640ab766 node DatanodeRegistration(127.0.0.1:37864, datanodeUuid=74100111-f3ba-4d97-bacc-ce9077eebf2a, infoPort=42399, infoSecurePort=0, ipcPort=47702, storageInfo=lv=-56;cid=testClusterID;nsid=661839495;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3172595 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38184
[junit4] 2> 3172758 INFO (IPC Server handler 4 on 39614) [ ] BlockStateChange BLOCK* processReport: from storage DS-1760dbda-8dae-4473-8df0-f0626a941231 node DatanodeRegistration(127.0.0.1:48356, datanodeUuid=a66ac63a-0311-453d-9013-5b87c36b456a, infoPort=58925, infoSecurePort=0, ipcPort=59874, storageInfo=lv=-56;cid=testClusterID;nsid=661839495;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
[junit4] 2> 3172758 INFO (IPC Server handler 4 on 39614) [ ] BlockStateChange BLOCK* processReport: from storage DS-cf159f4d-6b60-409e-b696-44b22920b9cf node DatanodeRegistration(127.0.0.1:48356, datanodeUuid=a66ac63a-0311-453d-9013-5b87c36b456a, infoPort=58925, infoSecurePort=0, ipcPort=59874, storageInfo=lv=-56;cid=testClusterID;nsid=661839495;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
[junit4] 2> 3173817 INFO (TEST-HdfsRecoveryZkTest.test-seed#[54BA97A2AEE84936]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 3173821 INFO (qtp1814268624-41161) [n:127.0.0.1:48041_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3173824 INFO (OverseerThreadFactory-8651-thread-1-processing-n:127.0.0.1:55454_solr) [n:127.0.0.1:55454_solr ] o.a.s.c.CreateCollectionCmd Create collection recoverytest
[junit4] 2> 3173936 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 3173936 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2
[junit4] 2> 3174048 INFO (zkCallback-9376-thread-2-processing-n:127.0.0.1:55454_solr) [n:127.0.0.1:55454_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3174048 INFO (zkCallback-9375-thread-1-processing-n:127.0.0.1:48041_solr) [n:127.0.0.1:48041_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3174965 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 3174965 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.4.0
[junit4] 2> 3174974 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema [recoverytest_shard1_replica2] Schema name=minimal
[junit4] 2> 3174974 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema [recoverytest_shard1_replica1] Schema name=minimal
[junit4] 2> 3174977 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3174977 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3174979 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica1' using configuration from collection recoverytest
[junit4] 2> 3174979 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 x:recoverytest_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'recoverytest_shard1_replica2' using configuration from collection recoverytest
[junit4] 2> 3175008 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:39614/data
[junit4] 2> 3175008 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://solaris-vm:39614/data
[junit4] 2> 3175008 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3175008 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 3175009 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [[recoverytest_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_54BA97A2AEE84936-001/tempDir-001/node1/recoverytest_shard1_replica2], dataDir=[hdfs://solaris-vm:39614/data/recoverytest/core_node1/data/]
[junit4] 2> 3175009 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [[recoverytest_shard1_replica1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsRecoveryZkTest_54BA97A2AEE84936-001/tempDir-001/node2/recoverytest_shard1_replica1], dataDir=[hdfs://solaris-vm:39614/data/recoverytest/core_node2/data/]
[junit4] 2> 3175011 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:39614/data/recoverytest/core_node1/data/snapshot_metadata
[junit4] 2> 3175011 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:39614/data/recoverytest/core_node2/data/snapshot_metadata
[junit4] 2> 3175049 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3175049 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3175049 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3175049 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3175049 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3175049 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3177435 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3177435 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3177454 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:39614/data/recoverytest/core_node1/data
[junit4] 2> 3177454 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:39614/data/recoverytest/core_node2/data
[junit4] 2> 3177495 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:39614/data/recoverytest/core_node2/data/index
[junit4] 2> 3177507 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3177508 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3177508 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3177537 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://solaris-vm:39614/data/recoverytest/core_node1/data/index
[junit4] 2> 3177545 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 3177545 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Block cache target memory usage, slab size of [16777216] will allocate [1] slabs and use ~[16777216] bytes
[junit4] 2> 3177545 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.HdfsDirectoryFactory Creating new single instance HDFS BlockCache
[junit4] 2> 3177635 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3177637 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.s.b.BlockDirectory Block cache on write is disabled
[junit4] 2> 3177885 INFO (IPC Server handler 4 on 39614) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37864 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-1760dbda-8dae-4473-8df0-f0626a941231:NORMAL:127.0.0.1:48356|RBW], ReplicaUC[[DISK]DS-c0252d28-0f8a-4f04-8f1c-8c85cb53463e:NORMAL:127.0.0.1:37864|FINALIZED]]} size 0
[junit4] 2> 3177892 INFO (IPC Server handler 4 on 39614) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48356 is added to blk_1073741826_1002 size 71
[junit4] 2> 3177939 INFO (IPC Server handler 0 on 39614) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48356 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-03be733f-f43a-4900-af2a-e72d640ab766:NORMAL:127.0.0.1:37864|RBW], ReplicaUC[[DISK]DS-1760dbda-8dae-4473-8df0-f0626a941231:NORMAL:127.0.0.1:48356|RBW]]} size 0
[junit4] 2> 3177949 INFO (IPC Server handler 8 on 39614) [ ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:37864 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-03be733f-f43a-4900-af2a-e72d640ab766:NORMAL:127.0.0.1:37864|RBW], ReplicaUC[[DISK]DS-1760dbda-8dae-4473-8df0-f0626a941231:NORMAL:127.0.0.1:48356|RBW]]} size 0
[junit4] 2> 3178029 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3178029 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3178030 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 3178032 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
[junit4] 2> 3178032 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3178032 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
[junit4] 2> 3178058 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3178058 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3178066 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3178066 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3178266 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d844e9c[recoverytest_shard1_replica1] main]
[junit4] 2> 3178268 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3178269 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3178271 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@259ff53a[recoverytest_shard1_replica2] main]
[junit4] 2> 3178276 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3178277 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1552084062328324096
[junit4] 2> 3178279 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3178279 INFO (searcherExecutor-8656-thread-1-processing-n:127.0.0.1:48041_solr x:recoverytest_shard1_replica1 s:shard1 c:recoverytest r:core_node2) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.SolrCore [recoverytest_shard1_replica1] Registered new searcher Searcher@6d844e9c[recoverytest_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3178280 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3178280 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 3178280 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1552084062331469824
[junit4] 2> 3178280 INFO (searcherExecutor-8657-thread-1-processing-n:127.0.0.1:55454_solr x:recoverytest_shard1_replica2 s:shard1 c:recoverytest r:core_node1) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] Registered new searcher Searcher@259ff53a[recoverytest_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3178290 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3178290 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3178290 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/
[junit4] 2> 3178290 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:48041/solr START replicas=[http://127.0.0.1:55454/solr/recoverytest_shard1_replica2/] nUpdates=100
[junit4] 2> 3178291 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.PeerSync PeerSync: core=recoverytest_shard1_replica1 url=http://127.0.0.1:48041/solr DONE. We have no versions. sync failed.
[junit4] 2> 3178294 INFO (qtp161014505-41151) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.S.Request [recoverytest_shard1_replica2] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 3178295 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 3178295 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 3178298 INFO (zkCallback-9376-thread-2-processing-n:127.0.0.1:55454_solr) [n:127.0.0.1:55454_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3178298 INFO (zkCallback-9375-thread-1-processing-n:127.0.0.1:48041_solr) [n:127.0.0.1:48041_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3178304 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/ shard1
[junit4] 2> 3178436 INFO (zkCallback-9376-thread-1-processing-n:127.0.0.1:55454_solr) [n:127.0.0.1:55454_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3178445 INFO (zkCallback-9375-thread-1-processing-n:127.0.0.1:48041_solr) [n:127.0.0.1:48041_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3178480 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3178495 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica1&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=4561
[junit4] 2> 3178593 INFO (zkCallback-9376-thread-3-processing-n:127.0.0.1:55454_solr) [n:127.0.0.1:55454_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3178595 INFO (zkCallback-9375-thread-1-processing-n:127.0.0.1:48041_solr) [n:127.0.0.1:48041_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3179299 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=recoverytest_shard1_replica2&action=CREATE&numShards=1&collection=recoverytest&shard=shard1&wt=javabin&version=2} status=0 QTime=5365
[junit4] 2> 3179315 INFO (qtp1814268624-41161) [n:127.0.0.1:48041_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3179417 INFO (zkCallback-9376-thread-3-processing-n:127.0.0.1:55454_solr) [n:127.0.0.1:55454_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3179417 INFO (zkCallback-9375-thread-1-processing-n:127.0.0.1:48041_solr) [n:127.0.0.1:48041_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/recoverytest/state.json] for collection [recoverytest] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3179836 INFO (OverseerCollectionConfigSetProcessor-97005253359042566-127.0.0.1:55454_solr-n_0000000000) [n:127.0.0.1:55454_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3180316 INFO (qtp1814268624-41161) [n:127.0.0.1:48041_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&maxShardsPerNode=1&name=recoverytest&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=6497
[junit4] 2> 3180320 INFO (TEST-HdfsRecoveryZkTest.test-seed#[54BA97A2AEE84936]) [ ] o.a.s.c.RecoveryZkTest Indexing 1350 documents
[junit4] 2> 3180424 INFO (qtp161014505-41172) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-0 (1552084064498876416)]} 0 50
[junit4] 2> 3180425 INFO (qtp161014505-41151) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-0 (1552084064498876417)]} 0 49
[junit4] 2> 3180427 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-0 (1552084064498876417)]} 0 80
[junit4] 2> 3180428 INFO (qtp1814268624-41177) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-0 (1552084064498876416)]} 0 82
[junit4] 2> 3180439 INFO (qtp161014505-41150) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[2-0 (-1552084064586956801)]} 0 5
[junit4] 2> 3180440 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{delete=[1-0 (-1552084064586956800)]} 0 7
[junit4] 2> 3180448 INFO (qtp1814268624-41167) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[2-0 (-1552084064586956801)]} 0 16
[junit4] 2> 3180448 INFO (qtp1814268624-41163) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{delete=[1-0 (-1552084064586956800)]} 0 17
[junit4] 2> 3180464 INFO (qtp161014505-41154) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-1 (1552084064614219776)]} 0 3
[junit4] 2> 3180465 INFO (qtp161014505-41157) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-1 (1552084064615268352)]} 0 5
[junit4] 2> 3180465 INFO (qtp1814268624-41162) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-1 (1552084064614219776)]} 0 8
[junit4] 2> 3180467 INFO (qtp1814268624-41165) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-1 (1552084064615268352)]} 0 9
[junit4] 2> 3180496 INFO (qtp161014505-41156) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-2 (1552084064637288449)]} 0 14
[junit4] 2> 3180497 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-2 (1552084064637288449)]} 0 17
[junit4] 2> 3180498 INFO (qtp161014505-41172) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-2 (1552084064637288448)]} 0 11
[junit4] 2> 3180498 INFO (qtp1814268624-41161) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-2 (1552084064637288448)]} 0 30
[junit4] 2> 3180506 INFO (qtp161014505-41151) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-3 (1552084064659308544)]} 0 2
[junit4] 2> 3180509 INFO (qtp161014505-41150) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-3 (1552084064662454272)]} 0 2
[junit4] 2> 3180510 INFO (qtp1814268624-41167) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-3 (1552084064662454272)]} 0 7
[junit4] 2> 3180511 INFO (qtp1814268624-41177) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-3 (1552084064659308544)]} 0 10
[junit4] 2> 3180526 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-4 (1552084064671891456)]} 0 12
[junit4] 2> 3180527 INFO (qtp1814268624-41163) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-4 (1552084064671891456)]} 0 14
[junit4] 2> 3180533 INFO (qtp161014505-41153) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-4 (1552084064672940032)]} 0 4
[junit4] 2> 3180534 INFO (qtp1814268624-41162) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-4 (1552084064672940032)]} 0 20
[junit4] 2> 3180547 ERROR (updateExecutor-9369-thread-2-processing-http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2 x:recoverytest_shard1_replica1 r:core_node2 n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.StreamingSolrClients error
[junit4] 2> java.net.SocketException: Connection reset
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:209)
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:141)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
[junit4] 2> at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180548 ERROR (updateExecutor-9369-thread-1-processing-http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2 x:recoverytest_shard1_replica1 r:core_node2 n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.StreamingSolrClients error
[junit4] 2> java.net.SocketException: Connection reset
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:209)
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:141)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
[junit4] 2> at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180549 INFO (TEST-HdfsRecoveryZkTest.test-seed#[54BA97A2AEE84936]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@24b4df37{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3180552 INFO (TEST-HdfsRecoveryZkTest.test-seed#[54BA97A2AEE84936]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=955272648
[junit4] 2> 3180554 WARN (qtp1814268624-41165) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Error sending update to http://127.0.0.1:55454/solr
[junit4] 2> java.net.SocketException: Connection reset
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:209)
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:141)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
[junit4] 2> at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180555 ERROR (qtp1814268624-41165) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on replica http://127.0.0.1:55454/solr/recoverytest_shard1_replica2/
[junit4] 2> java.net.SocketException: Connection reset
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:209)
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:141)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
[junit4] 2> at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180555 INFO (qtp1814268624-41165) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.ZkController Put replica core=recoverytest_shard1_replica2 coreNodeName=core_node1 on 127.0.0.1:55454_solr into leader-initiated recovery.
[junit4] 2> 3180555 INFO (qtp1814268624-41165) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-5 (1552084064697057280)]} 0 19
[junit4] 2> 3180561 INFO (updateExecutor-9369-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node2 http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2// n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread Put replica core=recoverytest_shard1_replica2 coreNodeName=core_node1 on 127.0.0.1:55454_solr into leader-initiated recovery.
[junit4] 2> 3180561 WARN (updateExecutor-9369-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node2 http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2// n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread Leader is publishing core=recoverytest_shard1_replica2 coreNodeName =core_node1 state=down on behalf of un-reachable replica http://127.0.0.1:55454/solr/recoverytest_shard1_replica2/
[junit4] 2> 3180562 INFO (updateExecutor-9369-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node2 http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2// n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread LeaderInitiatedRecoveryThread-recoverytest_shard1_replica2 started running to send REQUESTRECOVERY command to http://127.0.0.1:55454/solr/recoverytest_shard1_replica2/; will try for a max of 600 secs
[junit4] 2> 3180562 INFO (updateExecutor-9369-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node2 http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2// n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread Asking core=recoverytest_shard1_replica2 coreNodeName=core_node1 on http://127.0.0.1:55454/solr to recover
[junit4] 2> 3180565 ERROR (updateExecutor-9369-thread-1-processing-x:recoverytest_shard1_replica1 r:core_node2 http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2// n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.c.LeaderInitiatedRecoveryThread http://127.0.0.1:55454/solr: Could not tell a replica to recover:org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:55454/solr
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:600)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.cloud.LeaderInitiatedRecoveryThread.sendRecoveryCommandWithRetry(LeaderInitiatedRecoveryThread.java:208)
[junit4] 2> at org.apache.solr.cloud.LeaderInitiatedRecoveryThread.run(LeaderInitiatedRecoveryThread.java:95)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.net.ConnectException: Connection refused
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
[junit4] 2> ... 9 more
[junit4] 2>
[junit4] 2> 3180567 WARN (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Error sending update to http://127.0.0.1:55454/solr
[junit4] 2> java.net.SocketException: Connection reset
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:209)
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:141)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
[junit4] 2> at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180567 ERROR (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on replica http://127.0.0.1:55454/solr/recoverytest_shard1_replica2/
[junit4] 2> java.net.SocketException: Connection reset
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:209)
[junit4] 2> at java.net.SocketInputStream.read(SocketInputStream.java:141)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
[junit4] 2> at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
[junit4] 2> at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
[junit4] 2> at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
[junit4] 2> at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
[junit4] 2> at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
[junit4] 2> at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180567 INFO (qtp1814268624-41164) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1-5 (1552084064702300160)]} 0 27
[junit4] 2> 3180584 INFO (qtp161014505-41154) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[2-5 (1552084064697057280)]} 0 42
[junit4] 2> 3180584 INFO (qtp161014505-41157) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica2] webapp=/solr path=/update params={update.distrib=FROMLEADER&distrib.from=http://127.0.0.1:48041/solr/recoverytest_shard1_replica1/&wt=javabin&version=2}{add=[1-5 (1552084064702300160)]} 0 41
[junit4] 2> 3180585 INFO (qtp161014505-41157) [n:127.0.0.1:55454_solr c:recoverytest s:shard1 r:core_node1 x:recoverytest_shard1_replica2] o.a.s.c.SolrCore [recoverytest_shard1_replica2] CLOSING SolrCore org.apache.solr.core.SolrCore@4521ab24
[junit4] 2> 3180593 ERROR (updateExecutor-9369-thread-3-processing-http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2 x:recoverytest_shard1_replica1 r:core_node2 n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.StreamingSolrClients error
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180596 ERROR (updateExecutor-9369-thread-2-processing-http:////127.0.0.1:55454//solr//recoverytest_shard1_replica2 x:recoverytest_shard1_replica1 r:core_node2 n:127.0.0.1:48041_solr s:shard1 c:recoverytest) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.StreamingSolrClients error
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180599 WARN (qtp1814268624-41161) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Error sending update to http://127.0.0.1:55454/solr
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180599 ERROR (qtp1814268624-41161) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Setting up to try to start recovery on replica http://127.0.0.1:55454/solr/recoverytest_shard1_replica2/
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180599 INFO (qtp1814268624-41161) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.LogUpdateProcessorFactory [recoverytest_shard1_replica1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2-6 (1552084064747388928)]} 0 15
[junit4] 2> 3180600 WARN (qtp1814268624-41167) [n:127.0.0.1:48041_solr c:recoverytest s:shard1 r:core_node2 x:recoverytest_shard1_replica1] o.a.s.u.p.DistributedUpdateProcessor Error sending update to http://127.0.0.1:55454/solr
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4] 2> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
[junit4] 2> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
[junit4] 2> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
[junit4] 2> at java.net.Socket.connect(Socket.java:589)
[junit4] 2> at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:117)
[junit4] 2> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:177)
[junit4] 2> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:304)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:611)
[junit4] 2> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:446)
[junit4] 2> at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
[junit4] 2> at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.sendUpdateStream(ConcurrentUpdateSolrClient.java:312)
[junit4] 2> at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:185)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 3180600 ERRO
[...truncated too long message...]
java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> Caused by: java.lang.NullPointerException
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
[junit4] 2> at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
[junit4] 2> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
[junit4] 2> ... 54 more
[junit4] 2> 3199129 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:65350 65350
[junit4] 2> 3199130 INFO (SUITE-HdfsRecoveryZkTest-seed#[54BA97A2AEE84936]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true,coord=crazy): {}, locale=sr-Latn-RS, timezone=Europe/Busingen
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=3,free=182595504,total=435290112
[junit4] 2> NOTE: All tests run in this JVM: [TestSolrCloudWithSecureImpersonation, TestQueryTypes, HLLSerializationTest, TestInfoStreamLogging, TestSha256AuthenticationProvider, TestSuggestSpellingConverter, ConfigSetsAPITest, TestStandardQParsers, TestReRankQParserPlugin, TestSolrCoreProperties, HighlighterConfigTest, DistribJoinFromCollectionTest, CacheHeaderTest, ZkSolrClientTest, CdcrBootstrapTest, TestManagedResourceStorage, TestCollectionAPI, DistributedFacetPivotSmallTest, StatsComponentTest, MigrateRouteKeyTest, TestSolrQueryParserResource, DistributedTermsComponentTest, ChangedSchemaMergeTest, DocValuesMissingTest, ReplicaListTransformerTest, QueryResultKeyTest, PreAnalyzedUpdateProcessorTest, TestNonDefinedSimilarityFactory, TestExactSharedStatsCache, TestReloadDeadlock, ShardRoutingCustomTest, TestRandomFaceting, TestLMDirichletSimilarityFactory, CollectionTooManyReplicasTest, AutoCommitTest, TestSolrQueryParser, SmileWriterTest, TestRestoreCore, URLClassifyProcessorTest, TestSolrConfigHandlerConcurrent, TriLevelCompositeIdRoutingTest, DefaultValueUpdateProcessorTest, ResponseHeaderTest, CoreAdminRequestStatusTest, TestUtils, TestObjectReleaseTracker, TestCrossCoreJoin, TestLMJelinekMercerSimilarityFactory, SortByFunctionTest, TestDistributedGrouping, TestManagedStopFilterFactory, ScriptEngineTest, BadCopyFieldTest, TermVectorComponentTest, FieldMutatingUpdateProcessorTest, TestFiltering, TestDefaultStatsCache, AtomicUpdatesTest, TestFuzzyAnalyzedSuggestions, SolrIndexConfigTest, DistributedFacetPivotLongTailTest, TestFieldCacheSort, UtilsToolTest, HighlighterMaxOffsetTest, HdfsCollectionsAPIDistributedZkTest, InfoHandlerTest, LukeRequestHandlerTest, TestReplicationHandlerBackup, AddBlockUpdateTest, BasicDistributedZk2Test, TestSchemaSimilarityResource, TestFieldCacheReopen, TestSolrDynamicMBean, UUIDFieldTest, TestSolr4Spatial, HdfsRecoverLeaseTest, SpatialHeatmapFacetsTest, CircularListTest, TestReversedWildcardFilterFactory, UpdateParamsTest, TestConfigSetsAPIExclusivity, DistanceFunctionTest, TestNumericTerms64, SearchHandlerTest, TestLRUStatsCache, TestBadConfig, TestDynamicFieldCollectionResource, WordBreakSolrSpellCheckerTest, TestRecovery, SliceStateTest, DistributedSuggestComponentTest, TestLeaderElectionWithEmptyReplica, TestOnReconnectListenerSupport, TestRangeQuery, TestSolrQueryResponse, TestCollationFieldDocValues, SolrIndexSplitterTest, TestInitQParser, PeerSyncWithIndexFingerprintCachingTest, SignatureUpdateProcessorFactoryTest, TestGeoJSONResponseWriter, TestCustomDocTransformer, TestExpandComponent, LeaderInitiatedRecoveryOnCommitTest, RestartWhileUpdatingTest, TestQueryUtils, TestJavabinTupleStreamParser, LeaderElectionIntegrationTest, TestFieldCacheVsDocValues, SolrXmlInZkTest, DistributedDebugComponentTest, TestSolr4Spatial2, TestEmbeddedSolrServerConstructors, SolrInfoMBeanTest, FileBasedSpellCheckerTest, TestNamedUpdateProcessors, DirectUpdateHandlerOptimizeTest, FullSolrCloudDistribCmdsTest, BadIndexSchemaTest, TestFoldingMultitermQuery, TestReload, ExplicitHLLTest, TestCloudPivotFacet, TestTrieFacet, OverseerRolesTest, ResourceLoaderTest, FieldAnalysisRequestHandlerTest, TestTrackingShardHandlerFactory, TestPKIAuthenticationPlugin, CdcrUpdateLogTest, TestConfig, TestJoin, TestZkChroot, TestSimpleTrackingShardHandler, IgnoreCommitOptimizeUpdateProcessorFactoryTest, UUIDUpdateProcessorFallbackTest, DistributedVersionInfoTest, TestCorePropertiesReload, TestDynamicLoading, SparseHLLTest, DeleteInactiveReplicaTest, TestRawResponseWriter, LoggingHandlerTest, TestExceedMaxTermLength, TestUseDocValuesAsStored2, DebugComponentTest, TestDocTermOrds, CoreSorterTest, HdfsThreadLeakTest, TestAnalyzeInfixSuggestions, TestBulkSchemaConcurrent, TestDFRSimilarityFactory, BigEndianAscendingWordSerializerTest, TestJsonFacets, TestConfigSetProperties, TestCloudRecovery, DistributedIntervalFacetingTest, CleanupOldIndexTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, CreateCollectionCleanupTest, DeleteNodeTest, DeleteShardTest, DistribDocExpirationUpdateProcessorTest, ForceLeaderTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, RemoteQueryErrorTest, ReplicationFactorTest, SSLMigrationTest, SaslZkACLProviderTest, ShardSplitTest, TestShortCircuitedRequests, TestSolrCloudWithDelegationTokens, TestStressLiveNodes, HdfsRecoveryZkTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=54BA97A2AEE84936 -Dtests.slow=true -Dtests.locale=sr-Latn-RS -Dtests.timezone=Europe/Busingen -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | HdfsRecoveryZkTest (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
[junit4] > at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
[junit4] > at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:202)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:137)
[junit4] > at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:94)
[junit4] > at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:102)
[junit4] > at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
[junit4] > at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:705)
[junit4] > at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:767)
[junit4] > at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1006)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:871)
[junit4] > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:775)
[junit4] > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:842)
[junit4] > at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:498)
[junit4] > at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] > at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([54BA97A2AEE84936]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:260)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] Completed [533/657 (1!)] on J1 in 60.22s, 1 test, 1 failure <<< FAILURES!
[...truncated 61164 lines...]
[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 515 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/515/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
1 tests failed.
FAILED: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail
Error Message:
expected:<200> but was:<404>
Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
at __randomizedtesting.SeedInfo.seed([2E473BB1930558E0:46F80E9B439F4A0C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:141)
at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:295)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Build Log:
[...truncated 12211 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_2E473BB1930558E0-001/init-core-data-001
[junit4] 2> 2424942 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[2E473BB1930558E0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 2424946 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[2E473BB1930558E0]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_2E473BB1930558E0-001/tempDir-001
[junit4] 2> 2424947 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[2E473BB1930558E0]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2424947 INFO (Thread-6808) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2424947 INFO (Thread-6808) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2425047 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[2E473BB1930558E0]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:64747
[junit4] 2> 2425062 INFO (jetty-launcher-2990-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2425062 INFO (jetty-launcher-2990-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 2425065 INFO (jetty-launcher-2990-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1830a77b{/solr,null,AVAILABLE}
[junit4] 2> 2425066 INFO (jetty-launcher-2990-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c1e57d5{/solr,null,AVAILABLE}
[junit4] 2> 2425066 INFO (jetty-launcher-2990-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@73b85cfc{HTTP/1.1,[http/1.1]}{127.0.0.1:37200}
[junit4] 2> 2425066 INFO (jetty-launcher-2990-thread-2) [ ] o.e.j.s.Server Started @2429174ms
[junit4] 2> 2425066 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37200}
[junit4] 2> 2425066 ERROR (jetty-launcher-2990-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2425066 INFO (jetty-launcher-2990-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@541893c4{HTTP/1.1,[http/1.1]}{127.0.0.1:63656}
[junit4] 2> 2425067 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2425067 INFO (jetty-launcher-2990-thread-1) [ ] o.e.j.s.Server Started @2429175ms
[junit4] 2> 2425067 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2425067 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=63656}
[junit4] 2> 2425067 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2425067 ERROR (jetty-launcher-2990-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2425067 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-26T06:36:18.427Z
[junit4] 2> 2425068 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.4.0
[junit4] 2> 2425068 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2425068 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2425068 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2016-11-26T06:36:18.428Z
[junit4] 2> 2425071 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2425072 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2425082 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2425082 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 2425083 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64747/solr
[junit4] 2> 2425084 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64747/solr
[junit4] 2> 2425123 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:63656_solr
[junit4] 2> 2425124 INFO (jetty-launcher-2990-thread-2) [n:127.0.0.1:37200_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37200_solr
[junit4] 2> 2425125 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.s.c.Overseer Overseer (id=97002597797724165-127.0.0.1:63656_solr-n_0000000000) starting
[junit4] 2> 2425127 INFO (zkCallback-3001-thread-1-processing-n:127.0.0.1:37200_solr) [n:127.0.0.1:37200_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2425128 INFO (zkCallback-3000-thread-1-processing-n:127.0.0.1:63656_solr) [n:127.0.0.1:63656_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2425140 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63656_solr
[junit4] 2> 2425144 INFO (zkCallback-3000-thread-1-processing-n:127.0.0.1:63656_solr) [n:127.0.0.1:63656_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2425145 INFO (zkCallback-3001-thread-1-processing-n:127.0.0.1:37200_solr) [n:127.0.0.1:37200_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2425196 INFO (jetty-launcher-2990-thread-2) [n:127.0.0.1:37200_solr ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 2425206 INFO (jetty-launcher-2990-thread-2) [n:127.0.0.1:37200_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 2425279 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 2425279 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
[junit4] 2> 2425360 INFO (jetty-launcher-2990-thread-2) [n:127.0.0.1:37200_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 2425360 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 2425403 INFO (jetty-launcher-2990-thread-1-EventThread) [n:127.0.0.1:63656_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 2425405 INFO (jetty-launcher-2990-thread-2-EventThread) [n:127.0.0.1:37200_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 2425574 INFO (jetty-launcher-2990-thread-2) [n:127.0.0.1:37200_solr ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@38bc159b
[junit4] 2> 2425575 INFO (jetty-launcher-2990-thread-2) [n:127.0.0.1:37200_solr ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@38bc159b
[junit4] 2> 2425575 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.s.h.c.HttpShardHandlerFactory Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@47a5362d
[junit4] 2> 2425575 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.s.u.UpdateShardHandler Reconfiguring the default client with: org.apache.solr.security.HttpParamDelegationTokenPlugin$2@47a5362d
[junit4] 2> 2425580 INFO (jetty-launcher-2990-thread-1) [n:127.0.0.1:63656_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_2E473BB1930558E0-001/tempDir-001/node1/.
[junit4] 2> 2425587 INFO (jetty-launcher-2990-thread-2) [n:127.0.0.1:37200_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_2E473BB1930558E0-001/tempDir-001/node2/.
[junit4] 2> 2425621 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
[junit4] 2> 2425705 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2425707 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2425717 INFO (qtp1713939869-21405) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUq7igFYw2XOuwEEFKnYsl-BAe0f5pAm-P9zOwziJNrTB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425717 INFO (qtp1713939869-21405) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUq7igFYw2XOuwEEFKnYsl-BAe0f5pAm-P9zOwziJNrTB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425730 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425735 INFO (qtp447427768-21395) [n:127.0.0.1:37200_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUq7igFYw2XOuwEEFKnYsl-BAe0f5pAm-P9zOwziJNrTB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425735 INFO (qtp447427768-21395) [n:127.0.0.1:37200_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUq7igFYw2XOuwEEFKnYsl-BAe0f5pAm-P9zOwziJNrTB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425738 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425760 INFO (qtp1713939869-21391) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUscigFYw2XPHAIEFLko1CyQeENgAZ-wPEVQX_ZzZU9YB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425760 INFO (qtp1713939869-21391) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUscigFYw2XPHAIEFLko1CyQeENgAZ-wPEVQX_ZzZU9YB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425764 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2425766 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2425768 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425771 INFO (qtp447427768-21397) [n:127.0.0.1:37200_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUscigFYw2XPHAIEFLko1CyQeENgAZ-wPEVQX_ZzZU9YB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425771 INFO (qtp447427768-21397) [n:127.0.0.1:37200_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUscigFYw2XPHAIEFLko1CyQeENgAZ-wPEVQX_ZzZU9YB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425773 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425784 INFO (qtp1713939869-21403) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUs_igFYw2XPPwMEFFrQVcNXVg6QmeNJlzihM9MLt54fB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425784 INFO (qtp1713939869-21403) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUs_igFYw2XPPwMEFFrQVcNXVg6QmeNJlzihM9MLt54fB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425788 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2425790 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2425792 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425794 INFO (qtp447427768-21396) [n:127.0.0.1:37200_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUs_igFYw2XPPwMEFFrQVcNXVg6QmeNJlzihM9MLt54fB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425794 INFO (qtp447427768-21396) [n:127.0.0.1:37200_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUs_igFYw2XPPwMEFFrQVcNXVg6QmeNJlzihM9MLt54fB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425796 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425814 INFO (qtp1713939869-21402) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUtWigFYw2XPVgQEFGBPuj-HVOOIHa_CFY50Qc8i0ZTUB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425814 INFO (qtp1713939869-21402) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUtWigFYw2XPVgQEFGBPuj-HVOOIHa_CFY50Qc8i0ZTUB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425819 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2425820 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2425821 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425826 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2425827 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2425828 INFO (qtp1713939869-21404) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUtWigFYw2XPVgQEFGBPuj-HVOOIHa_CFY50Qc8i0ZTUB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425828 INFO (qtp1713939869-21404) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUtWigFYw2XPVgQEFGBPuj-HVOOIHa_CFY50Qc8i0ZTUB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425831 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1589f594806000e, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2425834 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2425835 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2425836 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425837 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
[junit4] 2> 2425885 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
[junit4] 2> 2425892 WARN (qtp1713939869-21405) [n:127.0.0.1:63656_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 2425893 WARN (qtp1713939869-21405) [n:127.0.0.1:63656_solr ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 2425893 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425895 WARN (qtp447427768-21396) [n:127.0.0.1:37200_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
[junit4] 2> 2425896 WARN (qtp447427768-21396) [n:127.0.0.1:37200_solr ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 2425896 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425903 WARN (qtp1713939869-21391) [n:127.0.0.1:63656_solr ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 2425903 WARN (qtp1713939869-21391) [n:127.0.0.1:63656_solr ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 2425903 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425905 WARN (qtp447427768-21398) [ ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
[junit4] 2> 2425905 WARN (qtp447427768-21398) [ ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
[junit4] 2> 2425905 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425905 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
[junit4] 2> 2425931 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
[junit4] 2> 2425952 INFO (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
[junit4] 2> 2425978 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
[junit4] 2> 2425987 INFO (qtp1713939869-21404) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUwMigFYw2XQDAgEFEGRhlBJMC-cvlQbUethS3uf1nSxB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425987 INFO (qtp1713939869-21404) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUwMigFYw2XQDAgEFEGRhlBJMC-cvlQbUethS3uf1nSxB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2425989 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2425992 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2425994 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2425995 INFO (qtp1713939869-21404) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUwMigFYw2XQDAgEFEGRhlBJMC-cvlQbUethS3uf1nSxB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2425995 INFO (qtp1713939869-21404) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUwMigFYw2XQDAgEFEGRhlBJMC-cvlQbUethS3uf1nSxB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2426000 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2426001 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2426003 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2426004 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
[junit4] 2> 2426032 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
[junit4] 2> 2426035 INFO (qtp1713939869-21403) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2426035 INFO (qtp1713939869-21403) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2426035 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1480178179394&s=Uco0+R2kKwyrz3PVyLlFyyjcRF4="; Path=/; Domain=127.0.0.1; Expires=Sht, 26-Nën-2016 16:36:19 GMT; HttpOnly". Invalid 'expires' attribute: Sht, 26-Nën-2016 16:36:19 GMT
[junit4] 2> 2426036 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
[junit4] 2> 2426057 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
[junit4] 2> 2426095 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=2E473BB1930558E0 -Dtests.slow=true -Dtests.locale=sq-AL -Dtests.timezone=America/North_Dakota/New_Salem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.06s J0 | TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
[junit4] > at __randomizedtesting.SeedInfo.seed([2E473BB1930558E0:46F80E9B439F4A0C]:0)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:141)
[junit4] > at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:295)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2426126 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
[junit4] 2> 2426136 ERROR (qtp1713939869-21405) [n:127.0.0.1:63656_solr ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_10]
[junit4] 2> 2426143 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2426145 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2426146 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2426150 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2426151 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2426152 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2426165 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2426166 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2426168 ERROR (qtp1713939869-21402) [n:127.0.0.1:63656_solr ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_11]
[junit4] 2> 2426174 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2426175 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2426176 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2426180 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2426181 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64747/solr ready
[junit4] 2> 2426183 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2426184 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
[junit4] 2> 2426209 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
[junit4] 2> 2426218 WARN (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[2E473BB1930558E0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires=Enj, 01-Jan-1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: Enj, 01-Jan-1970 00:00:00 GMT
[junit4] 2> 2426221 INFO (qtp1713939869-21408) [n:127.0.0.1:63656_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUzzigFYw2XQ8wwEFIAw00AOmCMagu3P14kIlbfqjzedB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2426221 INFO (qtp1713939869-21408) [n:127.0.0.1:63656_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVifWUzzigFYw2XQ8wwEFIAw00AOmCMagu3P14kIlbfqjzedB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2426222 INFO (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[2E473BB1930558E0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
[junit4] 2> 2426224 INFO (jetty-launcher-2990-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@541893c4{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2426224 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=334630654
[junit4] 2> 2426225 INFO (jetty-launcher-2990-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@73b85cfc{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2426227 INFO (jetty-launcher-2990-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1889406716
[junit4] 2> 2426232 INFO (jetty-launcher-2990-thread-2) [ ] o.a.s.c.Overseer Overseer (id=97002597797724165-127.0.0.1:63656_solr-n_0000000000) closing
[junit4] 2> 2426232 INFO (OverseerStateUpdate-97002597797724165-127.0.0.1:63656_solr-n_0000000000) [n:127.0.0.1:63656_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:63656_solr
[junit4] 2> 2426234 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1589f5948060005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2426237 ERROR (Thread[Thread-6825,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:63656_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 2426247 INFO (zkCallback-3001-thread-2-processing-n:127.0.0.1:37200_solr) [n:127.0.0.1:37200_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 2426248 INFO (zkCallback-3001-thread-1-processing-n:127.0.0.1:37200_solr) [n:127.0.0.1:37200_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37200_solr
[junit4] 2> 2426252 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1589f5948060007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 2426253 INFO (jetty-launcher-2990-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@c1e57d5{/solr,null,UNAVAILABLE}
[junit4] 2> 2427755 WARN (zkCallback-3001-thread-2-processing-n:127.0.0.1:37200_solr) [n:127.0.0.1:37200_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 2427756 ERROR (Thread[Thread-6826,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:37200_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 2427760 INFO (jetty-launcher-2990-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1830a77b{/solr,null,UNAVAILABLE}
[junit4] 2> 2427764 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[2E473BB1930558E0]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64747 64747
[junit4] 2> 2427790 INFO (Thread-6808) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64747 64747
[junit4] 2> 2427792 WARN (Thread-6808) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/aliases.json
[junit4] 2> 15 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/clusterstate.json
[junit4] 2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/live_nodes
[junit4] 2> 15 /solr/collections
[junit4] 2> 3 /solr/overseer/queue
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 2427792 INFO (SUITE-TestSolrCloudWithDelegationTokens-seed#[2E473BB1930558E0]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_2E473BB1930558E0-001
[junit4] 2> Nov 26, 2016 6:36:21 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Lucene62, sim=RandomSimilarity(queryNorm=true,coord=no): {}, locale=sq-AL, timezone=America/North_Dakota/New_Salem
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=3,threads=1,free=204054824,total=507834368
[junit4] 2> NOTE: All tests run in this JVM: [DocValuesTest, OutputWriterTest, DocValuesMultiTest, TestHighlightDedupGrouping, DistributedQueryElevationComponentTest, TermsComponentTest, TestDownShardTolerantSearch, SimpleCollectionCreateDeleteTest, SimpleMLTQParserTest, BlockCacheTest, JSONWriterTest, NumericFieldsTest, JavabinLoaderTest, TestCustomStream, PolyFieldTest, DistributedFacetPivotSmallAdvancedTest, TestDistributedStatsComponentCardinality, ForceLeaderTest, SpatialRPTFieldTypeTest, TestBlendedInfixSuggestions, TestSQLHandler, DirectSolrSpellCheckerTest, TestAuthenticationFramework, HdfsLockFactoryTest, TestFastLRUCache, TestXIncludeConfig, CdcrRequestHandlerTest, DocumentBuilderTest, RemoteQueryErrorTest, ZkNodePropsTest, TestStressReorder, SimplePostToolTest, XmlUpdateRequestHandlerTest, TestConfigSetImmutable, HdfsChaosMonkeySafeLeaderTest, TestSolrJ, LeaderElectionTest, StatelessScriptUpdateProcessorFactoryTest, BufferStoreTest, TestExtendedDismaxParser, ConcurrentDeleteAndCreateCollectionTest, TestRTimerTree, TestReloadAndDeleteDocs, SyncSliceTest, TestXmlQParser, TolerantUpdateProcessorTest, AssignTest, TestCustomSort, TestInitParams, TestCollationField, HttpSolrCallGetCoreTest, TestExactStatsCache, TestManagedSchemaThreadSafety, TestFieldCacheSanityChecker, HdfsWriteToMultipleCollectionsTest, TestHdfsCloudBackupRestore, TestJmxIntegration, TestPartialUpdateDeduplication, HdfsSyncSliceTest, UnloadDistributedZkTest, LeaderElectionContextKeyTest, TestGraphMLResponseWriter, TestImplicitCoreProperties, TestTolerantSearch, PluginInfoTest, HardAutoCommitTest, TestFaceting, TestBinaryResponseWriter, DeleteStatusTest, TestMiniSolrCloudClusterSSL, TestSSLRandomization, TestChildDocTransformer, HdfsRecoveryZkTest, SuggesterFSTTest, TestMiniSolrCloudCluster, DocumentAnalysisRequestHandlerTest, CollectionsAPIAsyncDistributedZkTest, TestCSVResponseWriter, DistributedMLTComponentTest, TestSortByMinMaxFunction, TimerUtilsTest, TestRequestForwarding, TestLeaderElectionZkExpiry, DirectoryFactoryTest, ShardRoutingTest, TestPostingsSolrHighlighter, TestSchemaResource, TestRawTransformer, TestQueryWrapperFilter, TestPushWriter, TestFastOutputStream, TestStressLucene, TestSolrFieldCacheMBean, TestPhraseSuggestions, SoftAutoCommitTest, TestFunctionQuery, TestCloudSchemaless, DirectSolrConnectionTest, TestManagedResource, TestSchemaVersionResource, TestRemoteStreaming, DistanceUnitsTest, TestFieldTypeResource, TestAddFieldRealTimeGet, TestPseudoReturnFields, MultiThreadedOCPTest, TestSort, ZkControllerTest, TestStressCloudBlindAtomicUpdates, TestElisionMultitermQuery, FullHLLTest, TestDistribIDF, TestDistributedSearch, VMParamsZkACLAndCredentialsProvidersTest, TestHdfsUpdateLog, SaslZkACLProviderTest, TestDocBasedVersionConstraints, OverseerStatusTest, CursorPagingTest, TestRebalanceLeaders, TestLazyCores, DateFieldTest, ShufflingReplicaListTransformerTest, AnalyticsMergeStrategyTest, DataDrivenBlockJoinTest, CollectionStateFormat2Test, TestRandomDVFaceting, HdfsDirectoryFactoryTest, PrimitiveFieldTypeTest, TestSolrConfigHandler, TestSubQueryTransformerCrossCore, TestWordDelimiterFilterFactory, TestDFISimilarityFactory, BlockJoinFacetDistribTest, TestUniqueKeyFieldResource, DistributedFacetPivotWhiteBoxTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestExportWriter, TestHdfsBackupRestoreCore, TestConfigSetsAPI, DistribDocExpirationUpdateProcessorTest, DistributedQueueTest, CoreMergeIndexesAdminHandlerTest, ReplicationFactorTest, TestReplicationHandler, TestSmileRequest, SystemInfoHandlerTest, TestFilteredDocIdSet, TestBlobHandler, TestUpdate, TestLFUCache, CdcrReplicationDistributedZkTest, TestUninvertingReader, BadComponentTest, QueryEqualityTest, CursorMarkTest, HighlighterTest, DirectUpdateHandlerTest, WrapperMergePolicyFactoryTest, SecurityConfHandlerTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, MultiTermTest, SampleTest, TestBinaryField, TestSearchPerf, ExternalFileFieldSortTest, TestSolrCoreProperties, SpellPossibilityIteratorTest, TestCharFilters, SynonymTokenizerTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TimeZoneUtilsTest, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, ChaosMonkeyNothingIsSafeTest, PrimUtilsTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, TestLRUCache, TestUtils, TestDocumentBuilder, FileUtilsTest, CircularListTest, TestRTGBase, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestJettySolrRunner, ConnectionReuseTest, ActionThrottleTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CdcrVersionReplicationTest, CollectionReloadTest, ConfigSetsAPITest, ConnectionManagerTest, CustomCollectionTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteShardTest, DistribJoinFromCollectionTest, HttpPartitionTest, OverseerModifyCollectionTest, RecoveryAfterSoftCommitTest, RollingRestartTest, ShardSplitTest, TestOnReconnectListenerSupport, TestRequestStatusCollectionAPI, TestSolrCloudWithDelegationTokens]
[junit4] Completed [507/657 (1!)] on J0 in 3.88s, 8 tests, 1 failure <<< FAILURES!
[...truncated 61313 lines...]