You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/07/23 18:43:52 UTC

[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 583 - Failure

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/583/

1 tests failed.
REGRESSION:  org.apache.lucene.analysis.core.TestRandomChains.testRandomChains

Error Message:
startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=11,endOffset=9

Stack Trace:
java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=11,endOffset=9
	at __randomizedtesting.SeedInfo.seed([E58A02D1D3CDCA0A:D86B2BB094DFD7CA]:0)
	at org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl.setOffset(PackedTokenAttributeImpl.java:107)
	at org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken(ShingleFilter.java:345)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
	at org.apache.lucene.analysis.synonym.SynonymFilter.parse(SynonymFilter.java:358)
	at org.apache.lucene.analysis.synonym.SynonymFilter.incrementToken(SynonymFilter.java:624)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:703)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:614)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:513)
	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChains(TestRandomChains.java:927)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 6257 lines...]
   [junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
   [junit4]   2> TEST FAIL: useCharFilter=false text='cyhvywqrlb piawukb'
   [junit4]   2> Exception from random analyzer: 
   [junit4]   2> charfilters=
   [junit4]   2>   org.apache.lucene.analysis.MockCharFilter(java.io.StringReader@57f37c49)
   [junit4]   2> tokenizer=
   [junit4]   2>   org.apache.lucene.analysis.path.PathHierarchyTokenizer(org.apache.lucene.analysis.core.TestRandomChains$CheckThatYouDidntReadAnythingReaderWrapper@76f82dd5, 0)
   [junit4]   2> filters=
   [junit4]   2>   org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter(LUCENE_4_10, ValidatingTokenFilter@69b98af4 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, [91, -47, 13, -116, 89, -19, 27, -11, 5, 24, 27, 38, -47, -21, -49, -7, -20, 94, -77, -76, -85, 55, 31, -26, 60, 40, 49, 3, 115, -95, -19, -62, -2, -17, 7, -2, -24, -44, 6, 110, -1, -6, 105, -96, -120, -15, -114, -91, 89, -30, 90, 3, 119, -128, 114, 113, -97, 109, 26, -104, -117, 87, 25, 123, -58, -93, 57, -86, -124, 50, -80, -83, 54, -37, 3, 41, -69, -58, 64, 116, -80, 11, -77, 125, -50, -94, 34, -14, -24, 72, -12, 51, 14, -101, -81, -60, 7, 43, 68, 96, -65], -43, [uvikyvgqiw, qoojdpui, yocw, mpots, fdhjherr, yug, rvkfgxw])
   [junit4]   2>   org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter(LUCENE_4_10, ValidatingTokenFilter@7073bf6 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, 9, [aytazlgaz, limttvfrfg, kxbl])
   [junit4]   2>   org.apache.lucene.analysis.shingle.ShingleFilter(ValidatingTokenFilter@41f5d969 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, ql)
   [junit4]   2>   org.apache.lucene.analysis.synonym.SynonymFilter(ValidatingTokenFilter@2226966d term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, org.apache.lucene.analysis.synonym.SynonymMap@135ddf9f, true)
   [junit4]   2> offsetsAreCorrect=false
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChains -Dtests.seed=E58A02D1D3CDCA0A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=ko_KR -Dtests.timezone=Asia/Brunei -Dtests.file.encoding=UTF-8
   [junit4] ERROR   34.5s J1 | TestRandomChains.testRandomChains <<<
   [junit4]    > Throwable #1: java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=11,endOffset=9
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E58A02D1D3CDCA0A:D86B2BB094DFD7CA]:0)
   [junit4]    > 	at org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl.setOffset(PackedTokenAttributeImpl.java:107)
   [junit4]    > 	at org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken(ShingleFilter.java:345)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
   [junit4]    > 	at org.apache.lucene.analysis.synonym.SynonymFilter.parse(SynonymFilter.java:358)
   [junit4]    > 	at org.apache.lucene.analysis.synonym.SynonymFilter.incrementToken(SynonymFilter.java:624)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:703)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:614)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:513)
   [junit4]    > 	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChains(TestRandomChains.java:927)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/build/analysis/common/test/J1/./lucene.analysis.core.TestRandomChains-E58A02D1D3CDCA0A-001
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=223), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=223)), sim=RandomSimilarityProvider(queryNorm=true,coord=no): {dummy=DFR I(ne)B1}, locale=ko_KR, timezone=Asia/Brunei
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=131689664,total=318767104
   [junit4]   2> NOTE: All tests run in this JVM: [TestCJKBigramFilterFactory, TestCodepointCountFilterFactory, TestLimitTokenPositionFilterFactory, TestGalicianStemFilter, TestCapitalizationFilter, TestCharTokenizers, TestPortugueseMinimalStemFilter, TestKeywordMarkerFilter, TestCollationKeyFilter, TestNorwegianMinimalStemFilter, DelimitedPayloadTokenFilterTest, DateRecognizerSinkTokenizerTest, TestSoraniNormalizationFilter, TestPortugueseLightStemFilterFactory, TestGermanStemFilterFactory, TestFinnishLightStemFilterFactory, TestPatternReplaceFilter, TestScandinavianNormalizationFilterFactory, TestStopFilter, TestTypeTokenFilterFactory, TestRandomChains]
   [junit4] Completed on J1 in 87.61s, 2 tests, 1 error <<< FAILURES!

[...truncated 651 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/build.xml:544: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:2019: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/analysis/build.xml:106: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/analysis/build.xml:38: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/module-build.xml:60: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 281 suites, 1466 tests, 1 error, 1 ignored

Total time: 102 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #582
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 1781114 bytes
Compression is 0.0%
Took 3 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 585 - Still Failing

Posted by Shalin Shekhar Mangar <sh...@gmail.com>.
I opened SOLR-6286 for the TestReplicationHandler failure.


On Sun, Jul 27, 2014 at 10:17 PM, Apache Jenkins Server <
jenkins@builds.apache.org> wrote:

> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/585/
>
> 3 tests failed.
> REGRESSION:
>  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload
>
> Error Message:
> expected:<[{indexVersion=1406477990053,generation=2,filelist=[_bta.fdt,
> _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim,
> _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _
> nik.si, _yok.cfe, _yok.cfs, _yok.si, _yp3.cfe, _yp3.cfs, _yp3.si,
> _yp4.cfe, _yp4.cfs, _yp4.si, _yp5.cfe, _yp5.cfs, _yp5.si, _yp6.cfe,
> _yp6.cfs, _yp6.si, _yp7.cfe, _yp7.cfs, _yp7.si, _yp8.cfe, _yp8.cfs, _
> yp8.si, _yp9.cfe, _yp9.cfs, _yp9.si, _ypa.cfe, _ypa.cfs, _ypa.si,
> _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe,
> _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _
> ypy.si, segments_2]}]> but
> was:<[{indexVersion=1406477990053,generation=2,filelist=[_bta.fdt,
> _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim,
> _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _
> nik.si, _yok.cfe, _yok.cfs, _yok.si, _yp3.cfe, _yp3.cfs, _yp3.si,
> _yp4.cfe, _yp4.cfs, _yp4.si, _yp5.cfe, _yp5.cfs, _yp5.si, _yp6.cfe,
> _yp6.cfs, _yp6.si, _yp7.cfe, _yp7.cfs, _yp7.si, _yp8.cfe, _yp8.cfs, _
> yp8.si, _yp9.cfe, _yp9.cfs, _yp9.si, _ypa.cfe, _ypa.cfs, _ypa.si,
> _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe,
> _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _
> ypy.si, segments_2]},
> {indexVersion=1406477990053,generation=3,filelist=[_bta.fdt, _bta.fdx,
> _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim,
> _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _
> nik.si, _ypc.cfe, _ypc.cfs, _ypc.si, _ypu.cfe, _ypu.cfs, _ypu.si,
> _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe, _ypw.cfs, _ypw.si, _ypx.cfe,
> _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _ypy.si, segments_3]}]>
>
> Stack Trace:
> java.lang.AssertionError:
> expected:<[{indexVersion=1406477990053,generation=2,filelist=[_bta.fdt,
> _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim,
> _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _
> nik.si, _yok.cfe, _yok.cfs, _yok.si, _yp3.cfe, _yp3.cfs, _yp3.si,
> _yp4.cfe, _yp4.cfs, _yp4.si, _yp5.cfe, _yp5.cfs, _yp5.si, _yp6.cfe,
> _yp6.cfs, _yp6.si, _yp7.cfe, _yp7.cfs, _yp7.si, _yp8.cfe, _yp8.cfs, _
> yp8.si, _yp9.cfe, _yp9.cfs, _yp9.si, _ypa.cfe, _ypa.cfs, _ypa.si,
> _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe,
> _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _
> ypy.si, segments_2]}]> but
> was:<[{indexVersion=1406477990053,generation=2,filelist=[_bta.fdt,
> _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim,
> _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _
> nik.si, _yok.cfe, _yok.cfs, _yok.si, _yp3.cfe, _yp3.cfs, _yp3.si,
> _yp4.cfe, _yp4.cfs, _yp4.si, _yp5.cfe, _yp5.cfs, _yp5.si, _yp6.cfe,
> _yp6.cfs, _yp6.si, _yp7.cfe, _yp7.cfs, _yp7.si, _yp8.cfe, _yp8.cfs, _
> yp8.si, _yp9.cfe, _yp9.cfs, _yp9.si, _ypa.cfe, _ypa.cfs, _ypa.si,
> _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe,
> _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _
> ypy.si, segments_2]},
> {indexVersion=1406477990053,generation=3,filelist=[_bta.fdt, _bta.fdx,
> _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim,
> _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _
> nik.si, _ypc.cfe, _ypc.cfs, _ypc.si, _ypu.cfe, _ypu.cfs, _ypu.si,
> _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe, _ypw.cfs, _ypw.si, _ypx.cfe,
> _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _ypy.si, segments_3]}]>
>         at
> __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC:C128D6FAFE8166BF]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.failNotEquals(Assert.java:647)
>         at org.junit.Assert.assertEquals(Assert.java:128)
>         at org.junit.Assert.assertEquals(Assert.java:147)
>         at
> org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1190)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at
> org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> FAILED:
>  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandlerBackup
>
> Error Message:
> 1 thread leaked from SUITE scope at
> org.apache.solr.handler.TestReplicationHandlerBackup:     1)
> Thread[id=31922, name=Thread-8018, state=RUNNABLE,
> group=TGRP-TestReplicationHandlerBackup]         at
> java.net.PlainSocketImpl.socketConnect(Native Method)         at
> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>         at
> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>         at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>   at java.net.Socket.connect(Socket.java:579)         at
> java.net.Socket.connect(Socket.java:528)         at
> sun.net.NetworkClient.doConnect(NetworkClient.java:180)         at
> sun.net.www.http.HttpClient.openServer(HttpClient.java:432)         at
> sun.net.www.http.HttpClient.openServer(HttpClient.java:527)         at
> sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)         at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
>         at java.net.URL.openStream(URL.java:1037)         at
> org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from
> SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup:
>    1) Thread[id=31922, name=Thread-8018, state=RUNNABLE,
> group=TGRP-TestReplicationHandlerBackup]
>         at java.net.PlainSocketImpl.socketConnect(Native Method)
>         at
> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>         at
> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>         at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>         at java.net.Socket.connect(Socket.java:579)
>         at java.net.Socket.connect(Socket.java:528)
>         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
>         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
>         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
>         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
>         at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
>         at java.net.URL.openStream(URL.java:1037)
>         at
> org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
>         at __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC]:0)
>
>
> FAILED:
>  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandlerBackup
>
> Error Message:
> There are still zombie threads that couldn't be terminated:    1)
> Thread[id=31922, name=Thread-8018, state=RUNNABLE,
> group=TGRP-TestReplicationHandlerBackup]         at
> java.net.PlainSocketImpl.socketConnect(Native Method)         at
> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>         at
> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>         at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>   at java.net.Socket.connect(Socket.java:579)         at
> java.net.Socket.connect(Socket.java:528)         at
> sun.net.NetworkClient.doConnect(NetworkClient.java:180)         at
> sun.net.www.http.HttpClient.openServer(HttpClient.java:432)         at
> sun.net.www.http.HttpClient.openServer(HttpClient.java:527)         at
> sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)         at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
>         at java.net.URL.openStream(URL.java:1037)         at
> org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie
> threads that couldn't be terminated:
>    1) Thread[id=31922, name=Thread-8018, state=RUNNABLE,
> group=TGRP-TestReplicationHandlerBackup]
>         at java.net.PlainSocketImpl.socketConnect(Native Method)
>         at
> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>         at
> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>         at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>         at java.net.Socket.connect(Socket.java:579)
>         at java.net.Socket.connect(Socket.java:528)
>         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
>         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
>         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
>         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
>         at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
>         at java.net.URL.openStream(URL.java:1037)
>         at
> org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
>         at __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC]:0)
>
>
>
>
> Build Log:
> [...truncated 12533 lines...]
>    [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
>    [junit4]   2> Creating dataDir:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/init-core-data-001
>    [junit4]   2> 4385109 T16350 oas.SolrTestCaseJ4.setUp ###Starting
> doTestDetails
>    [junit4]   2> 4385124 T16350 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 4385130 T16350 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:36483
>    [junit4]   2> 4385130 T16350 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>    [junit4]   2> 4385131 T16350 oasc.SolrResourceLoader.locateSolrHome
> JNDI not configured for solr (NoInitialContextEx)
>    [junit4]   2> 4385131 T16350 oasc.SolrResourceLoader.locateSolrHome
> using system property solr.solr.home:
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001
>    [junit4]   2> 4385131 T16350 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/'
>    [junit4]   2> 4385175 T16350 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/solr.xml
>    [junit4]   2> 4385253 T16350 oasc.CoreContainer.<init> New
> CoreContainer 248555160
>    [junit4]   2> 4385254 T16350 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/]
>    [junit4]   2> 4385255 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 4385256 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 4385256 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 4385256 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 4385257 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 4385257 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 4385257 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 4385258 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 4385258 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 4385259 T16350 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params:
> socketTimeout=340000&connTimeout=45000&retry=false
>    [junit4]   2> 4385259 T16350 oasl.LogWatcher.createWatcher SLF4J impl
> is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 4385259 T16350 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 4385260 T16350 oasc.CoreContainer.load Host Name:
> 127.0.0.1
>    [junit4]   2> 4385265 T16360 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/'
>    [junit4]   2> 4385314 T16360 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_4_10
>    [junit4]   2> 4385356 T16360 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>    [junit4]   2> 4385357 T16360 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>    [junit4]   2> 4385363 T16360 oass.IndexSchema.readSchema [collection1]
> Schema name=test
>    [junit4]   2> 4385380 T16360 oass.IndexSchema.readSchema unique key
> field: id
>    [junit4]   2> 4385381 T16360 oasc.CoreContainer.create Creating
> SolrCore 'collection1' using configuration from instancedir
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/
>    [junit4]   2> 4385382 T16360 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 4385382 T16360 oasc.SolrCore.<init> [collection1] Opening
> new SolrCore at
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/,
> dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/
>    [junit4]   2> 4385382 T16360 oasc.SolrCore.<init> JMX monitoring not
> detected for core: collection1
>    [junit4]   2> 4385383 T16360 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data
>    [junit4]   2> 4385383 T16360 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index/
>    [junit4]   2> 4385383 T16360 oasc.SolrCore.initIndex WARN [collection1]
> Solr index directory
> '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index'
> doesn't exist. Creating new index...
>    [junit4]   2> 4385384 T16360 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index
>    [junit4]   2> 4385384 T16360 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=26,
> maxMergedSegmentMB=67.6337890625, floorSegmentMB=1.693359375,
> forceMergeDeletesPctAllowed=19.688060269334418, segmentsPerTier=11.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5920792516815081
>    [junit4]   2> 4385385 T16360 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 4385385 T16360 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 1
>    [junit4]   2> 4385386 T16360 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 4385386 T16360
> oasc.RequestHandlers.initHandlersFromConfig created standard:
> solr.StandardRequestHandler
>    [junit4]   2> 4385387 T16360
> oasc.RequestHandlers.initHandlersFromConfig created /replication:
> solr.ReplicationHandler
>    [junit4]   2> 4385387 T16360
> oasc.RequestHandlers.initHandlersFromConfig created defaults:
> solr.StandardRequestHandler
>    [junit4]   2> 4385387 T16360
> oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler:
> solr.StandardRequestHandler
>    [junit4]   2> 4385387 T16360
> oasc.RequestHandlers.initHandlersFromConfig created lazy:
> solr.StandardRequestHandler
>    [junit4]   2> 4385388 T16360
> oasc.RequestHandlers.initHandlersFromConfig created /update:
> solr.UpdateRequestHandler
>    [junit4]   2> 4385406 T16360 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 4385409 T16360 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>    [junit4]   2> 4385409 T16360 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>    [junit4]   2> 4385409 T16360 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
> minMergeSize=1677721, mergeFactor=9, maxMergeSize=2147483648,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=false, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8239541096061591]
>    [junit4]   2> 4385410 T16360 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 4385410 T16360 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 1
>    [junit4]   2> 4385410 T16360 oass.SolrIndexSearcher.<init> Opening
> Searcher@76f2320d[collection1] main
>    [junit4]   2> 4385411 T16360
> oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage
> initialized to use dir:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
>    [junit4]   2> 4385411 T16360 oasr.RestManager.init Initializing
> RestManager with initArgs:
> {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf}
>    [junit4]   2> 4385411 T16360 oasr.ManagedResourceStorage.load Reading
> _rest_managed.json using
> file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
>    [junit4]   2> 4385412 T16360 oasr.ManagedResource.reloadFromStorage
> WARN No stored data found for /rest/managed
>    [junit4]   2> 4385413 T16360
> oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path
> _rest_managed.json using
> file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
>    [junit4]   2> 4385413 T16360 oasr.RestManager.init Initializing 0
> registered ManagedResources
>    [junit4]   2> 4385413 T16360 oash.ReplicationHandler.inform Replication
> enabled for following config files: schema.xml,xslt/dummy.xsl
>    [junit4]   2> 4385414 T16360 oash.ReplicationHandler.inform Commits
> will be reserved for  10000
>    [junit4]   2> 4385414 T16360 oasc.CoreContainer.registerCore
> registering core: collection1
>    [junit4]   2> 4385414 T16361 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@76f2320d[collection1]
> main{StandardDirectoryReader(segments_1:1:nrt)}
>    [junit4]   2> 4385414 T16350 oass.SolrDispatchFilter.init
> user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
>    [junit4]   2> 4385415 T16350 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>    [junit4]   2> 4385428 T16350 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 4385433 T16350 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:36484
>    [junit4]   2> 4385433 T16350 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>    [junit4]   2> 4385434 T16350 oasc.SolrResourceLoader.locateSolrHome
> JNDI not configured for solr (NoInitialContextEx)
>    [junit4]   2> 4385434 T16350 oasc.SolrResourceLoader.locateSolrHome
> using system property solr.solr.home:
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002
>    [junit4]   2> 4385434 T16350 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/'
>    [junit4]   2> 4385451 T16350 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/solr.xml
>    [junit4]   2> 4385495 T16350 oasc.CoreContainer.<init> New
> CoreContainer 1601195805
>    [junit4]   2> 4385495 T16350 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/]
>    [junit4]   2> 4385497 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 4385497 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 4385497 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 4385498 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 4385498 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 4385498 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 4385499 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 4385499 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 4385499 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 4385500 T16350 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params:
> socketTimeout=340000&connTimeout=45000&retry=false
>    [junit4]   2> 4385500 T16350 oasl.LogWatcher.createWatcher SLF4J impl
> is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 4385501 T16350 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 4385501 T16350 oasc.CoreContainer.load Host Name:
> 127.0.0.1
>    [junit4]   2> 4385506 T16372 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/'
>    [junit4]   2> 4385555 T16372 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_4_10
>    [junit4]   2> 4385609 T16372 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>    [junit4]   2> 4385610 T16372 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>    [junit4]   2> 4385616 T16372 oass.IndexSchema.readSchema [collection1]
> Schema name=test
>    [junit4]   2> 4385628 T16372 oass.IndexSchema.readSchema unique key
> field: id
>    [junit4]   2> 4385629 T16372 oasc.CoreContainer.create Creating
> SolrCore 'collection1' using configuration from instancedir
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/
>    [junit4]   2> 4385630 T16372 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 4385630 T16372 oasc.SolrCore.<init> [collection1] Opening
> new SolrCore at
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/,
> dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/
>    [junit4]   2> 4385630 T16372 oasc.SolrCore.<init> JMX monitoring not
> detected for core: collection1
>    [junit4]   2> 4385631 T16372 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data
>    [junit4]   2> 4385631 T16372 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index/
>    [junit4]   2> 4385631 T16372 oasc.SolrCore.initIndex WARN [collection1]
> Solr index directory
> '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index'
> doesn't exist. Creating new index...
>    [junit4]   2> 4385633 T16372 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index
>    [junit4]   2> 4385633 T16372 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=26,
> maxMergedSegmentMB=67.6337890625, floorSegmentMB=1.693359375,
> forceMergeDeletesPctAllowed=19.688060269334418, segmentsPerTier=11.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5920792516815081
>    [junit4]   2> 4385634 T16372 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 4385635 T16372 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 1
>    [junit4]   2> 4385635 T16372 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 4385636 T16372
> oasc.RequestHandlers.initHandlersFromConfig created standard:
> solr.StandardRequestHandler
>    [junit4]   2> 4385636 T16372
> oasc.RequestHandlers.initHandlersFromConfig created defaults:
> solr.StandardRequestHandler
>    [junit4]   2> 4385636 T16372
> oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler:
> solr.StandardRequestHandler
>    [junit4]   2> 4385636 T16372
> oasc.RequestHandlers.initHandlersFromConfig created lazy:
> solr.StandardRequestHandler
>    [junit4]   2> 4385637 T16372
> oasc.RequestHandlers.initHandlersFromConfig created /update:
> solr.UpdateRequestHandler
>    [junit4]   2> 4385637 T16372
> oasc.RequestHandlers.initHandlersFromConfig created /replication:
> solr.ReplicationHandler
>    [junit4]   2> 4385639 T16372 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 4385641 T16372 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>    [junit4]   2> 4385641 T16372 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>    [junit4]   2> 4385642 T16372 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
> minMergeSize=1677721, mergeFactor=9, maxMergeSize=2147483648,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=false, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8239541096061591]
>    [junit4]   2> 4385642 T16372 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 4385643 T16372 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 1
>    [junit4]   2> 4385643 T16372 oass.SolrIndexSearcher.<init> Opening
> Searcher@542a0e5a[collection1] main
>    [junit4]   2> 4385643 T16372
> oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage
> initialized to use dir:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/conf
>    [junit4]   2> 4385644 T16372 oasr.RestManager.init Initializing
> RestManager with initArgs:
> {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/conf}
>    [junit4]   2> 4385644 T16372 oasr.ManagedResourceStorage.load Reading
> _rest_managed.json using
> file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/conf
>    [junit4]   2> 4385645 T16372 oasr.ManagedResource.reloadFromStorage
> WARN No stored data found for /rest/managed
>    [junit4]   2> 4385645 T16372
> oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path
> _rest_managed.json using
> file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/conf
>    [junit4]   2> 4385646 T16372 oasr.RestManager.init Initializing 0
> registered ManagedResources
>    [junit4]   2> 4385646 T16372 oash.SnapPuller.startExecutorService Poll
> Scheduled at an interval of 1000ms
>    [junit4]   2> 4385647 T16372 oasc.CoreContainer.registerCore
> registering core: collection1
>    [junit4]   2> 4385647 T16373 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@542a0e5a[collection1]
> main{StandardDirectoryReader(segments_1:1:nrt)}
>    [junit4]   2> 4385647 T16350 oass.SolrDispatchFilter.init
> user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
>    [junit4]   2> 4385648 T16350 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>    [junit4]   2> ASYNC  NEW_CORE C9906 name=collection1
> org.apache.solr.core.SolrCore@3ca433db
>    [junit4]   2> 4385651 T16353 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/select params={sort=id+desc&version=2&q=*:*&wt=javabin}
> hits=0 status=0 QTime=1
>    [junit4]   2> 4385654 T16353 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin}
> status=0 QTime=0
>    [junit4]   2> 4385659 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&slave=false&command=details&version=2&wt=javabin}
> status=0 QTime=0
>    [junit4]   2> ASYNC  NEW_CORE C9907 name=collection1
> org.apache.solr.core.SolrCore@1966442d
>    [junit4]   2> 4385660 T16365 C9907 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin}
> status=0 QTime=4
>    [junit4]   2> 4385663 T16353 C9906 oasup.LogUpdateProcessor.finish
> [collection1] webapp=/solr path=/update params={version=2&wt=javabin}
> {add=[0]} 0 0
>    [junit4]   2> 4385666 T16353 C9906 oasu.DirectUpdateHandler2.commit
> start
> commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
>    [junit4]   2> 4385667 T16353 C9906 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=2
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_1,generation=1}
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_2,generation=2}
>    [junit4]   2> 4385669 T16353 C9906
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
>    [junit4]   2> 4385671 T16353 C9906 oass.SolrIndexSearcher.<init>
> Opening Searcher@422aca31[collection1] main
>    [junit4]   2> 4385671 T16353 C9906 oasu.DirectUpdateHandler2.commit
> end_commit_flush
>    [junit4]   2> 4385671 T16361 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@422aca31[collection1]
> main{StandardDirectoryReader(segments_2:3:nrt _0(4.10):c1)}
>    [junit4]   2> 4385672 T16353 C9906 oasup.LogUpdateProcessor.finish
> [collection1] webapp=/solr path=/update
> params={waitSearcher=true&version=2&softCommit=false&commit=true&wt=javabin}
> {commit=} 0 6
>    [junit4]   2> 4385721 T16376 oash.SnapPuller.<init> WARN 'masterUrl'
> must be specified without the /replication suffix
>    [junit4]   2> 4385723 T16376 oash.SnapPuller.<init>  No value set for
> 'pollInterval'. Timer Task not started.
>    [junit4]   2> 4385726 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4385727 T16376 oash.SnapPuller.fetchLatestIndex Master's
> generation: 2
>    [junit4]   2> 4385727 T16376 oash.SnapPuller.fetchLatestIndex Slave's
> generation: 1
>    [junit4]   2> 4385727 T16376 oash.SnapPuller.fetchLatestIndex Starting
> replication process
>    [junit4]   2> 4385732 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filelist&version=2&generation=2&wt=javabin}
> status=0 QTime=2
>    [junit4]   2> 4385732 T16376 oash.SnapPuller.fetchLatestIndex Number of
> files in latest index in master: 4
>    [junit4]   2> 4385733 T16376 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919782
>    [junit4]   2> 4385733 T16376 oasu.DefaultSolrCoreState.closeIndexWriter
> Closing IndexWriter...
>    [junit4]   2> 4385733 T16376 oasu.DefaultSolrCoreState.closeIndexWriter
> Waiting until IndexWriter is unused... core=collection1
>    [junit4]   2> 4385734 T16376
> oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter...
> core=collection1
>    [junit4]   2> 4385734 T16376 oash.SnapPuller.fetchLatestIndex Starting
> download to MockDirectoryWrapper(RAMDirectory@27b3a72
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6f171dfe)
> fullCopy=false
>    [junit4]   2> 4385737 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=2&wt=filestream&file=_0.cfe}
> status=0 QTime=0
>    [junit4]   2> 4385741 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=2&wt=filestream&file=_
> 0.si} status=0 QTime=0
>    [junit4]   2> 4385744 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=2&wt=filestream&file=segments_2}
> status=0 QTime=0
>    [junit4]   2> 4385748 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=2&wt=filestream&file=_0.cfs}
> status=0 QTime=0
>    [junit4]   2> 4385748 T16376 oash.SnapPuller.fetchLatestIndex Total
> time taken for download : 0 secs
>    [junit4]   2> 4385750 T16376 oasu.DefaultSolrCoreState.openIndexWriter
> Creating new IndexWriter...
>    [junit4]   2> 4385750 T16376 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.MockRandomMergePolicy:
> org.apache.lucene.index.MockRandomMergePolicy@191825
>    [junit4]   2> 4385751 T16376 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=2
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33
> ),segFN=segments_1,generation=1}
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33
> ),segFN=segments_2,generation=2}
>    [junit4]   2> 4385753 T16376 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 2
>    [junit4]   2> 4385753 T16376 oasu.DefaultSolrCoreState.openIndexWriter
> New IndexWriter is ready to be used.
>    [junit4]   2> 4385754 T16376 oass.SolrIndexSearcher.<init> Opening
> Searcher@433a3fe2[collection1] main
>    [junit4]   2> 4385754 T16373 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@433a3fe2[collection1]
> main{StandardDirectoryReader(segments_2:3:nrt _0(4.10):c1)}
>    [junit4]   2> 4385755 T16376
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919782
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919782;done=true>>]
>    [junit4]   2> 4385755 T16376 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919782
>    [junit4]   2> 4385756 T16366 C9907 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication params={masterUrl=
> http://127.0.0.1:36483/solr/replication&command=fetchindex&wait=true}
> status=0 QTime=35
>    [junit4]   2> 4385766 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&slave=false&command=details&version=2&wt=javabin}
> status=0 QTime=0
>    [junit4]   2> 4385766 T16365 C9907 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin}
> status=0 QTime=2
>    [junit4]   2> 4385769 T16353 C9906 oasup.LogUpdateProcessor.finish
> [collection1] webapp=/solr path=/update params={version=2&wt=javabin}
> {add=[1]} 0 0
>    [junit4]   2> 4385773 T16353 C9906 oasu.DirectUpdateHandler2.commit
> start
> commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
>    [junit4]   2> 4385775 T16353 C9906 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=2
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_2,generation=2}
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_3,generation=3}
>    [junit4]   2> 4385775 T16353 C9906
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
>    [junit4]   2> 4385776 T16353 C9906 oass.SolrIndexSearcher.<init>
> Opening Searcher@5c29df30[collection1] main
>    [junit4]   2> 4385776 T16353 C9906 oasu.DirectUpdateHandler2.commit
> end_commit_flush
>    [junit4]   2> 4385776 T16361 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@5c29df30[collection1]
> main{StandardDirectoryReader(segments_3:5:nrt _0(4.10):c1 _1(4.10):c1)}
>    [junit4]   2> 4385777 T16353 C9906 oasup.LogUpdateProcessor.finish
> [collection1] webapp=/solr path=/update
> params={waitSearcher=true&version=2&softCommit=false&commit=true&wt=javabin}
> {commit=} 0 4
>    [junit4]   2> 4385778 T16379 oash.SnapPuller.<init> WARN 'masterUrl'
> must be specified without the /replication suffix
>    [junit4]   2> 4385779 T16379 oash.SnapPuller.<init>  No value set for
> 'pollInterval'. Timer Task not started.
>    [junit4]   2> 4385781 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4385781 T16379 oash.SnapPuller.fetchLatestIndex Master's
> generation: 3
>    [junit4]   2> 4385781 T16379 oash.SnapPuller.fetchLatestIndex Slave's
> generation: 2
>    [junit4]   2> 4385782 T16379 oash.SnapPuller.fetchLatestIndex Starting
> replication process
>    [junit4]   2> 4385784 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filelist&version=2&generation=3&wt=javabin}
> status=0 QTime=0
>    [junit4]   2> 4385785 T16379 oash.SnapPuller.fetchLatestIndex Number of
> files in latest index in master: 7
>    [junit4]   2> 4385785 T16379 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919834
>    [junit4]   2> 4385787 T16379 oasu.DefaultSolrCoreState.closeIndexWriter
> Closing IndexWriter...
>    [junit4]   2> 4385787 T16379 oasu.DefaultSolrCoreState.closeIndexWriter
> Waiting until IndexWriter is unused... core=collection1
>    [junit4]   2> 4385788 T16379
> oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter...
> core=collection1
>    [junit4]   2> 4385789 T16379 oash.SnapPuller.fetchLatestIndex Starting
> download to MockDirectoryWrapper(RAMDirectory@12661766
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@16d95102)
> fullCopy=false
>    [junit4]   2> 4385789 T16379 oash.SnapPuller.downloadIndexFiles
> Skipping download for _0.cfe because it already exists
>    [junit4]   2> 4385792 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=3&wt=filestream&file=_1.cfs}
> status=0 QTime=0
>    [junit4]   2> 4385796 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=3&wt=filestream&file=_1.cfe}
> status=0 QTime=0
>    [junit4]   2> 4385799 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=3&wt=filestream&file=segments_3}
> status=0 QTime=0
>    [junit4]   2> 4385799 T16379 oash.SnapPuller.downloadIndexFiles
> Skipping download for _0.si because it already exists
>    [junit4]   2> 4385800 T16379 oash.SnapPuller.downloadIndexFiles
> Skipping download for _0.cfs because it already exists
>    [junit4]   2> 4385802 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=3&wt=filestream&file=_
> 1.si} status=0 QTime=0
>    [junit4]   2> 4385802 T16379 oash.SnapPuller.fetchLatestIndex Total
> time taken for download : 0 secs
>    [junit4]   2> 4385804 T16379 oasu.DefaultSolrCoreState.openIndexWriter
> Creating new IndexWriter...
>    [junit4]   2> 4385804 T16379 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.MockRandomMergePolicy:
> org.apache.lucene.index.MockRandomMergePolicy@780b21e4
>    [junit4]   2> 4385805 T16379 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=2
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33
> ),segFN=segments_2,generation=2}
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33
> ),segFN=segments_3,generation=3}
>    [junit4]   2> 4385806 T16379 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 3
>    [junit4]   2> 4385806 T16379 oasu.DefaultSolrCoreState.openIndexWriter
> New IndexWriter is ready to be used.
>    [junit4]   2> 4385807 T16379 oass.SolrIndexSearcher.<init> Opening
> Searcher@54f1656a[collection1] main
>    [junit4]   2> 4385807 T16373 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@54f1656a[collection1]
> main{StandardDirectoryReader(segments_3:5:nrt _0(4.10):c1 _1(4.10):c1)}
>    [junit4]   2> 4385808 T16379
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919834
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919834;done=true>>]
>    [junit4]   2> 4385808 T16379 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919834
>    [junit4]   2> 4385808 T16366 C9907 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication params={masterUrl=
> http://127.0.0.1:36483/solr/replication&command=fetchindex&wait=true}
> status=0 QTime=30
>    [junit4]   2> 4385813 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&slave=false&command=details&version=2&wt=javabin}
> status=0 QTime=0
>    [junit4]   2> 4385814 T16365 C9907 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin}
> status=0 QTime=3
>    [junit4]   2> 4385816 T16353 C9906 oasup.LogUpdateProcessor.finish
> [collection1] webapp=/solr path=/update params={version=2&wt=javabin}
> {add=[2]} 0 0
>    [junit4]   2> 4385819 T16353 C9906 oasu.DirectUpdateHandler2.commit
> start
> commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
>    [junit4]   2> 4385821 T16353 C9906 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=3
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_2,generation=2}
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_3,generation=3}
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d
> ),segFN=segments_4,generation=4}
>    [junit4]   2> 4385821 T16353 C9906
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
>    [junit4]   2> 4385822 T16353 C9906 oass.SolrIndexSearcher.<init>
> Opening Searcher@10e0c2[collection1] main
>    [junit4]   2> 4385822 T16353 C9906 oasu.DirectUpdateHandler2.commit
> end_commit_flush
>    [junit4]   2> 4385822 T16361 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@10e0c2[collection1]
> main{StandardDirectoryReader(segments_4:7:nrt _0(4.10):c1 _1(4.10):c1
> _2(4.10):c1)}
>    [junit4]   2> 4385823 T16353 C9906 oasup.LogUpdateProcessor.finish
> [collection1] webapp=/solr path=/update
> params={waitSearcher=true&version=2&softCommit=false&commit=true&wt=javabin}
> {commit=} 0 4
>    [junit4]   2> 4385824 T16381 oash.SnapPuller.<init> WARN 'masterUrl'
> must be specified without the /replication suffix
>    [junit4]   2> 4385825 T16381 oash.SnapPuller.<init>  No value set for
> 'pollInterval'. Timer Task not started.
>    [junit4]   2> 4385827 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4385827 T16381 oash.SnapPuller.fetchLatestIndex Master's
> generation: 4
>    [junit4]   2> 4385827 T16381 oash.SnapPuller.fetchLatestIndex Slave's
> generation: 3
>    [junit4]   2> 4385828 T16381 oash.SnapPuller.fetchLatestIndex Starting
> replication process
>    [junit4]   2> 4385831 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filelist&version=2&generation=4&wt=javabin}
> status=0 QTime=0
>    [junit4]   2> 4385832 T16381 oash.SnapPuller.fetchLatestIndex Number of
> files in latest index in master: 10
>    [junit4]   2> 4385832 T16381 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919881
>    [junit4]   2> 4385833 T16381 oasu.DefaultSolrCoreState.closeIndexWriter
> Closing IndexWriter...
>    [junit4]   2> 4385833 T16381 oasu.DefaultSolrCoreState.closeIndexWriter
> Waiting until IndexWriter is unused... core=collection1
>    [junit4]   2> 4385833 T16381
> oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter...
> core=collection1
>    [junit4]   2> 4385835 T16381 oash.SnapPuller.fetchLatestIndex Starting
> download to MockDirectoryWrapper(RAMDirectory@500b9d3b
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3390947e)
> fullCopy=false
>    [junit4]   2> 4385838 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=4&wt=filestream&file=_
> 2.si} status=0 QTime=0
>    [junit4]   2> 4385839 T16381 oash.SnapPuller.downloadIndexFiles
> Skipping download for _0.cfe because it already exists
>    [junit4]   2> 4385839 T16381 oash.SnapPuller.downloadIndexFiles
> Skipping download for _1.cfs because it already exists
>    [junit4]   2> 4385839 T16381 oash.SnapPuller.downloadIndexFiles
> Skipping download for _1.cfe because it already exists
>    [junit4]   2> 4385842 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=4&wt=filestream&file=_2.cfe}
> status=0 QTime=0
>    [junit4]   2> 4385845 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=4&wt=filestream&file=segments_4}
> status=0 QTime=0
>    [junit4]   2> 4385845 T16381 oash.SnapPuller.downloadIndexFiles
> Skipping download for _0.si because it already exists
>    [junit4]   2> 4385846 T16381 oash.SnapPuller.downloadIndexFiles
> Skipping download for _0.cfs because it already exists
>    [junit4]   2> 4385848 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=filecontent&checksum=true&generation=4&wt=filestream&file=_2.cfs}
> status=0 QTime=0
>    [junit4]   2> 4385900 T16381 oash.SnapPuller.downloadIndexFiles
> Skipping download for _1.si because it already exists
>    [junit4]   2> 4385901 T16381 oash.SnapPuller.fetchLatestIndex Total
> time taken for download : 0 secs
>    [junit4]   2> 4385902 T16381 oasu.DefaultSolrCoreState.openIndexWriter
> Creating new IndexWriter...
>    [junit4]   2> 4385903 T16381 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.MockRandomMergePolicy:
> org.apache.lucene.index.MockRandomMergePolicy@3c84c37a
>    [junit4]   2> 4385904 T16381 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=2
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33
> ),segFN=segments_3,generation=3}
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33
> ),segFN=segments_4,generation=4}
>    [junit4]   2> 4385904 T16381 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 4
>    [junit4]   2> 4385904 T16381 oasu.DefaultSolrCoreState.openIndexWriter
> New IndexWriter is ready to be used.
>    [junit4]   2> 4385906 T16381 oass.SolrIndexSearcher.<init> Opening
> Searcher@1036b53[collection1] main
>    [junit4]   2> 4385906 T16373 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@1036b53[collection1]
> main{StandardDirectoryReader(segments_4:7:nrt _0(4.10):c1 _1(4.10):c1
> _2(4.10):c1)}
>    [junit4]   2> 4385906 T16381
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919881
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919881;done=true>>]
>    [junit4]   2> 4385907 T16381 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919881
>    [junit4]   2> 4385907 T16366 C9907 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication params={masterUrl=
> http://127.0.0.1:36483/solr/replication&command=fetchindex&wait=true}
> status=0 QTime=83
>    [junit4]   2> 4385923 T16350 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 4385929 T16350 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:36489
>    [junit4]   2> 4385930 T16350 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>    [junit4]   2> 4385930 T16350 oasc.SolrResourceLoader.locateSolrHome
> JNDI not configured for solr (NoInitialContextEx)
>    [junit4]   2> 4385930 T16350 oasc.SolrResourceLoader.locateSolrHome
> using system property solr.solr.home:
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003
>    [junit4]   2> 4385931 T16350 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/'
>    [junit4]   2> 4385955 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4385956 T16374 oash.SnapPuller.fetchLatestIndex Slave in
> sync with master.
>    [junit4]   2> 4385977 T16350 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/solr.xml
>    [junit4]   2> 4386058 T16350 oasc.CoreContainer.<init> New
> CoreContainer 1726482995
>    [junit4]   2> 4386058 T16350 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/]
>    [junit4]   2> 4386060 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 4386060 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 4386060 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 4386061 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 4386061 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 4386061 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 4386062 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 4386062 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 4386062 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 4386063 T16350 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params:
> socketTimeout=340000&connTimeout=45000&retry=false
>    [junit4]   2> 4386063 T16350 oasl.LogWatcher.createWatcher SLF4J impl
> is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 4386064 T16350 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 4386064 T16350 oasc.CoreContainer.load Host Name:
> 127.0.0.1
>    [junit4]   2> 4386069 T16392 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/'
>    [junit4]   2> 4386120 T16392 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_4_10
>    [junit4]   2> 4386162 T16392 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>    [junit4]   2> 4386163 T16392 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>    [junit4]   2> 4386169 T16392 oass.IndexSchema.readSchema [collection1]
> Schema name=test
>    [junit4]   2> 4386186 T16392 oass.IndexSchema.readSchema unique key
> field: id
>    [junit4]   2> 4386187 T16392 oasc.CoreContainer.create Creating
> SolrCore 'collection1' using configuration from instancedir
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/
>    [junit4]   2> 4386188 T16392 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 4386188 T16392 oasc.SolrCore.<init> [collection1] Opening
> new SolrCore at
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/,
> dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/
>    [junit4]   2> 4386188 T16392 oasc.SolrCore.<init> JMX monitoring not
> detected for core: collection1
>    [junit4]   2> 4386189 T16392 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data
>    [junit4]   2> 4386189 T16392 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index/
>    [junit4]   2> 4386189 T16392 oasc.SolrCore.initIndex WARN [collection1]
> Solr index directory
> '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index'
> doesn't exist. Creating new index...
>    [junit4]   2> 4386190 T16392 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index
>    [junit4]   2> 4386190 T16392 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=26,
> maxMergedSegmentMB=67.6337890625, floorSegmentMB=1.693359375,
> forceMergeDeletesPctAllowed=19.688060269334418, segmentsPerTier=11.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5920792516815081
>    [junit4]   2> 4386191 T16392 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@1e0e6a08
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@be33180
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 4386191 T16392 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 1
>    [junit4]   2> 4386192 T16392 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 4386193 T16392
> oasc.RequestHandlers.initHandlersFromConfig created standard:
> solr.StandardRequestHandler
>    [junit4]   2> 4386193 T16392
> oasc.RequestHandlers.initHandlersFromConfig created defaults:
> solr.StandardRequestHandler
>    [junit4]   2> 4386193 T16392
> oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler:
> solr.StandardRequestHandler
>    [junit4]   2> 4386193 T16392
> oasc.RequestHandlers.initHandlersFromConfig created lazy:
> solr.StandardRequestHandler
>    [junit4]   2> 4386193 T16392
> oasc.RequestHandlers.initHandlersFromConfig created /update:
> solr.UpdateRequestHandler
>    [junit4]   2> 4386194 T16392
> oasc.RequestHandlers.initHandlersFromConfig created /replication:
> solr.ReplicationHandler
>    [junit4]   2> 4386212 T16392 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 4386214 T16392 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>    [junit4]   2> 4386215 T16392 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>    [junit4]   2> 4386215 T16392 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
> minMergeSize=1677721, mergeFactor=9, maxMergeSize=2147483648,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=false, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8239541096061591]
>    [junit4]   2> 4386216 T16392 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@1e0e6a08
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@be33180
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 4386216 T16392 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 1
>    [junit4]   2> 4386216 T16392 oass.SolrIndexSearcher.<init> Opening
> Searcher@b696276[collection1] main
>    [junit4]   2> 4386217 T16392
> oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage
> initialized to use dir:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/conf
>    [junit4]   2> 4386217 T16392 oasr.RestManager.init Initializing
> RestManager with initArgs:
> {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/conf}
>    [junit4]   2> 4386217 T16392 oasr.ManagedResourceStorage.load Reading
> _rest_managed.json using
> file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/conf
>    [junit4]   2> 4386218 T16392 oasr.ManagedResource.reloadFromStorage
> WARN No stored data found for /rest/managed
>    [junit4]   2> 4386218 T16392
> oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path
> _rest_managed.json using
> file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/conf
>    [junit4]   2> 4386219 T16392 oasr.RestManager.init Initializing 0
> registered ManagedResources
>    [junit4]   2> 4386219 T16392 oash.SnapPuller.<init> WARN 'masterUrl'
> must be specified without the /replication suffix
>    [junit4]   2> 4386219 T16392 oash.SnapPuller.<init>  No value set for
> 'pollInterval'. Timer Task not started.
>    [junit4]   2> 4386220 T16392 oash.ReplicationHandler.inform Replication
> enabled for following config files: schema.xml
>    [junit4]   2> 4386220 T16392 oash.ReplicationHandler.inform Commits
> will be reserved for  10000
>    [junit4]   2> 4386220 T16392 oasc.CoreContainer.registerCore
> registering core: collection1
>    [junit4]   2> 4386220 T16393 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@b696276[collection1]
> main{StandardDirectoryReader(segments_1:1:nrt)}
>    [junit4]   2> 4386221 T16350 oass.SolrDispatchFilter.init
> user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
>    [junit4]   2> 4386221 T16350 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>    [junit4]   2> 4386227 T16355 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&slave=false&command=details&version=2&wt=javabin}
> status=0 QTime=0
>    [junit4]   2> ASYNC  NEW_CORE C9908 name=collection1
> org.apache.solr.core.SolrCore@698ce178
>    [junit4]   2> 4386227 T16385 C9908 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin}
> status=0 QTime=4
>    [junit4]   2> 4386228 T16350 oasc.CoreContainer.shutdown Shutting down
> CoreContainer instance=1726482995
>    [junit4]   2> 4386230 T16350 oasc.SolrCore.close [collection1]  CLOSING
> SolrCore org.apache.solr.core.SolrCore@698ce178
>    [junit4]   2> 4386230 T16350 oasu.DirectUpdateHandler2.close closing
> DirectUpdateHandler2{commits=0,autocommits=0,soft
> autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
>    [junit4]   2> 4386231 T16350 oasu.SolrCoreState.decrefSolrCoreState
> Closing SolrCoreState
>    [junit4]   2> 4386231 T16350 oasu.DefaultSolrCoreState.closeIndexWriter
> SolrCoreState ref count has reached 0 - closing IndexWriter
>    [junit4]   2> 4386232 T16350 oasu.DefaultSolrCoreState.closeIndexWriter
> closing IndexWriter with IndexWriterCloser
>    [junit4]   2> 4386232 T16350 oasc.SolrCore.closeSearcher [collection1]
> Closing main searcher on request.
>    [junit4]   2> 4386233 T16350 oasc.CachingDirectoryFactory.close Closing
> MockDirectoryFactory - 2 directories currently being tracked
>    [junit4]   2> 4386233 T16350
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data;done=false>>]
>    [junit4]   2> 4386234 T16350 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data
>    [junit4]   2> 4386234 T16350
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index;done=false>>]
>    [junit4]   2> 4386234 T16350 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index
>    [junit4]   2> 4386235 T16350 oejsh.ContextHandler.doStop stopped
> o.e.j.s.ServletContextHandler{/solr,null}
>    [junit4]   2> 4386954 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4386955 T16374 oash.SnapPuller.fetchLatestIndex Slave in
> sync with master.
>    [junit4]   2> 4387955 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4387955 T16374 oash.SnapPuller.fetchLatestIndex Slave in
> sync with master.
>    [junit4]   2> 4388956 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4388956 T16374 oash.SnapPuller.fetchLatestIndex Slave in
> sync with master.
>    [junit4]   2> 4389955 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4389956 T16374 oash.SnapPuller.fetchLatestIndex Slave in
> sync with master.
>    [junit4]   2> 4390956 T16354 C9906 oasc.SolrCore.execute [collection1]
> webapp=/solr path=/replication
> params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0
> QTime=0
>    [junit4]   2> 4390956 T16374 oash.SnapPuller.fetchLatestIndex Slave in
> sync with master.
>    [junit4]   2> 4391238 T16350 oas.SolrTestCaseJ4.tearDown ###Ending
> doTestDetails
>    [junit4]   2> 4391238 T16350 oasc.CoreContainer.shutdown Shutting down
> CoreContainer instance=248555160
>    [junit4]   2> 4391241 T16350 oasc.SolrCore.close [collection1]  CLOSING
> SolrCore org.apache.solr.core.SolrCore@3ca433db
>    [junit4]   2> 4391241 T16350 oasu.DirectUpdateHandler2.close closing
> DirectUpdateHandler2{commits=3,autocommits=0,soft
> autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
>    [junit4]   2> 4391244 T16350 oasu.SolrCoreState.decrefSolrCoreState
> Closing SolrCoreState
>    [junit4]   2> 4391245 T16350 oasu.DefaultSolrCoreState.closeIndexWriter
> SolrCoreState ref count has reached 0 - closing IndexWriter
>    [junit4]   2> 4391245 T16350 oasu.DefaultSolrCoreState.closeIndexWriter
> closing IndexWriter with IndexWriterCloser
>    [junit4]   2> 4391248 T16350 oasc.SolrCore.closeSearcher [collection1]
> Closing main searcher on request.
>    [junit4]   2> 4391249 T16350 oasc.CachingDirectoryFactory.close Closing
> MockDirectoryFactory - 2 directories currently being tracked
>    [junit4]   2> 4391249 T16350
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data;done=false>>]
>    [junit4]   2> 4391249 T16350 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data
>    [junit4]   2> 4391250 T16350
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index;done=false>>]
>    [junit4]   2> 4391250 T16350 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index
>    [junit4]   2> 4391251 T16350 oejsh.ContextHandler.doStop stopped
> o.e.j.s.ServletContextHandler{/solr,null}
>    [junit4]   2> 4396245 T16350 oasc.CoreContainer.shutdown Shutting down
> CoreContainer instance=1601195805
>    [junit4]   2> 4396247 T16350 oasc.SolrCore.close [collection1]  CLOSING
> SolrCore org.apache.solr.core.SolrCore@1966442d
>    [junit4]   2> 4406954 T16374 oash.SnapPuller.fetchLatestIndex ERROR
> Master at: http://127.0.0.1:36483/solr is not available. Index fetch
> failed. Exception: IOException occured when talking to server at:
> http://127.0.0.1:36483/solr
>    [junit4]   2> 4406957 T16350 oasu.DirectUpdateHandler2.close closing
> DirectUpdateHandler2{commits=0,autocommits=0,soft
> autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
>    [junit4]   2> 4406957 T16350 oasu.SolrCoreState.decrefSolrCoreState
> Closing SolrCoreState
>    [junit4]   2> 4406958 T16350 oasu.DefaultSolrCoreState.closeIndexWriter
> SolrCoreState ref count has reached 0 - closing IndexWriter
>    [junit4]   2> 4406958 T16350 oasu.DefaultSolrCoreState.closeIndexWriter
> closing IndexWriter with IndexWriterCloser
>    [junit4]   2> 4406960 T16350 oasc.SolrCore.closeSearcher [collection1]
> Closing main searcher on request.
>    [junit4]   2> 4406961 T16350 oasc.CachingDirectoryFactory.close Closing
> MockDirectoryFactory - 2 directories currently being tracked
>    [junit4]   2> 4406961 T16350
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data;done=false>>]
>    [junit4]   2> 4406962 T16350 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data
>    [junit4]   2> 4406962 T16350
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index;done=false>>]
>    [junit4]   2> 4406962 T16350 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index
>    [junit4]   2> 4406963 T16350 oejsh.ContextHandler.doStop stopped
> o.e.j.s.ServletContextHandler{/solr,null}
>    [junit4]   2> 4411271 T16350 oas.SolrTestCaseJ4.setUp ###Starting
> doTestReplicateAfterStartupWithNoActivity
>    [junit4]   2> 4411286 T16350 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 4411291 T16350 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:36495
>    [junit4]   2> 4411292 T16350 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>    [junit4]   2> 4411292 T16350 oasc.SolrResourceLoader.locateSolrHome
> JNDI not configured for solr (NoInitialContextEx)
>    [junit4]   2> 4411292 T16350 oasc.SolrResourceLoader.locateSolrHome
> using system property solr.solr.home:
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001
>    [junit4]   2> 4411293 T16350 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/'
>    [junit4]   2> 4411336 T16350 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/solr.xml
>    [junit4]   2> 4411429 T16350 oasc.CoreContainer.<init> New
> CoreContainer 1715233452
>    [junit4]   2> 4411430 T16350 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/]
>    [junit4]   2> 4411431 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 4411431 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 4411432 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 4411432 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 4411432 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 4411433 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 4411433 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 4411433 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 4411434 T16350 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 4411434 T16350 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params:
> socketTimeout=340000&connTimeout=45000&retry=false
>    [junit4]   2> 4411435 T16350 oasl.LogWatcher.createWatcher SLF4J impl
> is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 4411435 T16350 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 4411435 T16350 oasc.CoreContainer.load Host Name:
> 127.0.0.1
>    [junit4]   2> 4411440 T16404 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/'
>    [junit4]   2> 4411490 T16404 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_4_10
>    [junit4]   2> 4411532 T16404 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>    [junit4]   2> 4411532 T16404 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>    [junit4]   2> 4411539 T16404 oass.IndexSchema.readSchema [collection1]
> Schema name=test
>    [junit4]   2> 4411556 T16404 oass.IndexSchema.readSchema unique key
> field: id
>    [junit4]   2> 4411557 T16404 oasc.CoreContainer.create Creating
> SolrCore 'collection1' using configuration from instancedir
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/
>    [junit4]   2> 4411558 T16404 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 4411558 T16404 oasc.SolrCore.<init> [collection1] Opening
> new SolrCore at
> ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/,
> dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/
>    [junit4]   2> 4411558 T16404 oasc.SolrCore.<init> JMX monitoring not
> detected for core: collection1
>    [junit4]   2> 4411559 T16404 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data
>    [junit4]   2> 4411559 T16404 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index/
>    [junit4]   2> 4411559 T16404 oasc.SolrCore.initIndex WARN [collection1]
> Solr index directory
> '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index'
> doesn't exist. Creating new index...
>    [junit4]   2> 4411560 T16404 oasc.CachingDirectoryFactory.get return
> new directory for
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index
>    [junit4]   2> 4411560 T16404 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=26,
> maxMergedSegmentMB=67.6337890625, floorSegmentMB=1.693359375,
> forceMergeDeletesPctAllowed=19.688060269334418, segmentsPerTier=11.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5920792516815081
>    [junit4]   2> 4411561 T16404 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@794627d4
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4368d6a4
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 4411561 T16404 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 1
>    [junit4]   2> 4411562 T16404 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 4411562 T16404
> oasc.RequestHandlers.initHandlersFromConfig created standard:
> solr.StandardRequestHandler
>    [junit4]   2> 4411563 T16404
> oasc.RequestHandlers.initHandlersFromConfig created /replication:
> solr.ReplicationHandler
>    [junit4]   2> 4411563 T16404
> oasc.RequestHandlers.initHandlersFromConfig created defaults:
> solr.StandardRequestHandler
>    [junit4]   2> 4411563 T16404
> oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler:
> solr.StandardRequestHandler
>    [junit4]   2> 4411563 T16404
> oasc.RequestHandlers.initHandlersFromConfig created lazy:
> solr.StandardRequestHandler
>    [junit4]   2> 4411564 T16404
> oasc.RequestHandlers.initHandlersFromConfig created /update:
> solr.UpdateRequestHandler
>    [junit4]   2> 4411582 T16404 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 4411585 T16404 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>    [junit4]   2> 4411585 T16404 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>    [junit4]   2> 4411585 T16404 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
> minMergeSize=1677721, mergeFactor=9, maxMergeSize=2147483648,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=false, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8239541096061591]
>    [junit4]   2> 4411586 T16404 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>
>  commit{dir=MockDirectoryWrapper(RAMDirectory@794627d4
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4368d6a4
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 4411586 T16404 oasc.SolrDeletionPolicy.updateCommits
> newest commit generation = 1
>    [junit4]   2> 4411586 T16404 oass.SolrIndexSearcher.<init> Opening
> Searcher@c13b75b[collection1] main
>    [junit4]   2> 4411587 T16404
> oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage
> initialized to use dir:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
>    [junit4]   2> 4411587 T16404 oasr.RestManager.init Initializing
> RestManager with initArgs:
> {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf}
>    [junit4]   2> 4411587 T16404 oasr.ManagedResourceStorage.load Reading
> _rest_managed.json using
> file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
>    [junit4]   2> 4411588 T16404 oasr.ManagedResource.reloadFromStorage
> WARN No stored data found for /rest/managed
>    [junit4]   2> 4411589 T16404
> oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path
> _rest_managed.json using
> file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
>    [junit4]   2> 4411589 T16404 oasr.RestManager.init Initializing 0
> registered ManagedResources
>    [junit4]   2> 4411589 T16404 oash.ReplicationHandler.inform Replication
> enabled for following config files: schema.xml,xslt/dummy.xsl
>    [junit4]   2> 4411589 T16404 oash.ReplicationHandler.inform Commits
> will be reserved for  10000
>    [junit4]   2> 4411590 T16404 oasc.CoreContainer.registerCore
> registering core: collection1
>    [junit4]   2> 4411590 T16405 oasc.SolrCore.registerSearcher
> [collection1] Registered new searcher Searcher@c13b75b[collection1]
> main{StandardDirectoryReader(segments_1:1:nrt)}
>    [junit4]   2> 4411590 T16350 oas
>
> [...truncated too long message...]
>
> ieldUpdateProcessorsTest, TestMiniSolrCloudCluster,
> TestBlendedInfixSuggestions, ResponseLogComponentTest,
> SpellPossibilityIteratorTest, AnalyticsQueryTest, BlockCacheTest,
> OverseerRolesTest, TestCodecSupport, ReplicationFactorTest,
> ClusterStateUpdateTest, DocValuesMultiTest, EnumFieldTest,
> TestDistributedSearch, UnloadDistributedZkTest,
> CoreMergeIndexesAdminHandlerTest, TestLMJelinekMercerSimilarityFactory,
> ExternalFileFieldSortTest, TestFastOutputStream, InfoHandlerTest, TestSort,
> TestManagedSchema, TestDocBasedVersionConstraints, ShardSplitTest,
> TestStressUserVersions, TestShardHandlerFactory,
> TestDistribDocBasedVersion, CachingDirectoryFactoryTest, SuggesterTest,
> PingRequestHandlerTest, TestCollationFieldDocValues,
> TestFuzzyAnalyzedSuggestions, DistributedExpandComponentTest,
> CustomCollectionTest, DeleteInactiveReplicaTest,
> TestManagedSchemaFieldResource, TestFreeTextSuggestions, DeleteReplicaTest,
> TestHighFrequencyDictionaryFactory, ReturnFieldsTest,
> HdfsBasicDistributedZkTest, TestSolrQueryParserDefaultOperatorResource,
> TestRangeQuery, TestCopyFieldCollectionResource, TestSurroundQueryParser,
> WordBreakSolrSpellCheckerTest, SpellCheckCollatorTest,
> DocValuesMissingTest, TestSearchPerf, TestStressRecovery, TestReload,
> TestConfig, BasicZkTest, LeaderElectionTest, ZkSolrClientTest,
> TestZkChroot, ZkCLITest, ShardRoutingCustomTest, AutoCommitTest,
> TestDistributedGrouping, TestFaceting, TestHashPartitioner,
> DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest,
> ZkControllerTest, TestStressReorder, TestMultiCoreConfBootstrap,
> TestStressVersions, HardAutoCommitTest, SimpleFacetsTest, TestSolr4Spatial,
> SpellCheckComponentTest, TestGroupingSearch, PeerSyncTest,
> BadIndexSchemaTest, TestFunctionQuery, BasicFunctionalityTest,
> TestLazyCores, DirectUpdateHandlerTest, TestBadConfig,
> OverseerCollectionProcessorTest, ShowFileRequestHandlerTest,
> CurrencyFieldOpenExchangeTest, SimplePostToolTest, TestCoreDiscovery,
> AnalysisAfterCoreReloadTest, SuggesterFSTTest, CoreAdminHandlerTest,
> SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTSTTest,
> SuggesterWFSTTest, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest,
> SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest,
> TestAtomicUpdateErrorCases, QueryEqualityTest,
> DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest,
> SortByFunctionTest, TestSolrDeletionPolicy1, DebugComponentTest,
> DocumentAnalysisRequestHandlerTest, TestQueryTypes, PrimitiveFieldTypeTest,
> FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest,
> MoreLikeThisHandlerTest, RequiredFieldsTest,
> FieldAnalysisRequestHandlerTest, TestSolrQueryParser, LoggingHandlerTest,
> SolrPluginUtilsTest, IndexSchemaTest, TestJmxIntegration,
> TestAnalyzedSuggestions, TestPartialUpdateDeduplication,
> TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestComponentsName,
> CoreContainerCoreInitFailuresTest, TestBinaryResponseWriter, TestLFUCache,
> HighlighterConfigTest, SOLR749Test, TestQuerySenderListener,
> AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery,
> SolrIndexConfigTest, BadComponentTest, TestSolrDeletionPolicy2,
> MultiTermTest, TestDocSet, TestBinaryField, NumericFieldsTest,
> MinimalSchemaTest, TestElisionMultitermQuery, TestSolrCoreProperties,
> TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, TestCharFilters,
> SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest,
> TestDFRSimilarityFactory, TestNumberUtils, TestIBSimilarityFactory,
> TestBM25SimilarityFactory, TestDefaultSimilarityFactory, TestFastWriter,
> LegacyHTMLStripCharFilterTest, TestJmxMonitoredMap, TimeZoneUtilsTest,
> ResourceLoaderTest, OpenExchangeRatesOrgProviderTest,
> URLClassifyProcessorTest, PluginInfoTest, DateMathParserTest,
> ChaosMonkeyNothingIsSafeTest, TestSolrXMLSerializer, TestSystemIdResolver,
> TestSuggestSpellingConverter, DateFieldTest, RAMDirectoryFactoryTest,
> TestSolrJ, TestUtils, ZkNodePropsTest, SliceStateTest,
> SystemInfoHandlerTest, UUIDFieldTest, TestRTGBase, CursorPagingTest,
> SolrTestCaseJ4Test, TestDistributedMissingSort, TestTolerantSearch,
> AliasIntegrationTest, ConnectionManagerTest, DeleteShardTest,
> DistribCursorPagingTest, HttpPartitionTest, MigrateRouteKeyTest,
> TestShortCircuitedRequests, HdfsChaosMonkeySafeLeaderTest,
> HdfsRecoveryZkTest, HdfsSyncSliceTest, HdfsWriteToMultipleCollectionsTest,
> StressHdfsTest, TestInfoStreamLogging, TestNRTOpen,
> TestReloadAndDeleteDocs, TestSolrXml, TestSolrXmlPersistence,
> TestReplicationHandlerBackup]
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by
> running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test
>  -Dtestcase=TestReplicationHandlerBackup -Dtests.seed=E4FFCDCA8EC968BC
> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true
> -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt
> -Dtests.locale=sr_BA_#Latn -Dtests.timezone=America/Edmonton
> -Dtests.file.encoding=US-ASCII
>    [junit4] ERROR   0.00s J0 | TestReplicationHandlerBackup (suite) <<<
>    [junit4]    > Throwable #1:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from
> SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup:
>    [junit4]    >    1) Thread[id=31922, name=Thread-8018, state=RUNNABLE,
> group=TGRP-TestReplicationHandlerBackup]
>    [junit4]    >         at java.net.PlainSocketImpl.socketConnect(Native
> Method)
>    [junit4]    >         at
> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>    [junit4]    >         at
> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>    [junit4]    >         at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>    [junit4]    >         at
> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>    [junit4]    >         at java.net.Socket.connect(Socket.java:579)
>    [junit4]    >         at java.net.Socket.connect(Socket.java:528)
>    [junit4]    >         at
> sun.net.NetworkClient.doConnect(NetworkClient.java:180)
>    [junit4]    >         at
> sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
>    [junit4]    >         at
> sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
>    [junit4]    >         at
> sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
>    [junit4]    >         at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
>    [junit4]    >         at java.net.URL.openStream(URL.java:1037)
>    [junit4]    >         at
> org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
>    [junit4]    >        at
> __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC]:0)Throwable #2:
> com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie
> threads that couldn't be terminated:
>    [junit4]    >    1) Thread[id=31922, name=Thread-8018, state=RUNNABLE,
> group=TGRP-TestReplicationHandlerBackup]
>    [junit4]    >         at java.net.PlainSocketImpl.socketConnect(Native
> Method)
>    [junit4]    >         at
> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
>    [junit4]    >         at
> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
>    [junit4]    >         at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
>    [junit4]    >         at
> java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>    [junit4]    >         at java.net.Socket.connect(Socket.java:579)
>    [junit4]    >         at java.net.Socket.connect(Socket.java:528)
>    [junit4]    >         at
> sun.net.NetworkClient.doConnect(NetworkClient.java:180)
>    [junit4]    >         at
> sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
>    [junit4]    >         at
> sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
>    [junit4]    >         at
> sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
>    [junit4]    >         at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
>    [junit4]    >         at java.net.URL.openStream(URL.java:1037)
>    [junit4]    >         at
> org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
>    [junit4]    >        at
> __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC]:0)
>    [junit4] Completed on J0 in 30.12s, 1 test, 2 errors <<< FAILURES!
>
> [...truncated 135 lines...]
> BUILD FAILED
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923:
> There were test failures: 410 suites, 1702 tests, 2 suite-level errors, 1
> failure, 136 ignored (8 assumptions)
>
> Total time: 246 minutes 18 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #582
> Archived 3 artifacts
> Archive block size is 32768
> Received 0 blocks and 687356687 bytes
> Compression is 0.0%
> Took 1 min 48 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>



-- 
Regards,
Shalin Shekhar Mangar.

[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 585 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/585/

3 tests failed.
REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1406477990053,generation=2,filelist=[_bta.fdt, _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim, _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _nik.si, _yok.cfe, _yok.cfs, _yok.si, _yp3.cfe, _yp3.cfs, _yp3.si, _yp4.cfe, _yp4.cfs, _yp4.si, _yp5.cfe, _yp5.cfs, _yp5.si, _yp6.cfe, _yp6.cfs, _yp6.si, _yp7.cfe, _yp7.cfs, _yp7.si, _yp8.cfe, _yp8.cfs, _yp8.si, _yp9.cfe, _yp9.cfs, _yp9.si, _ypa.cfe, _ypa.cfs, _ypa.si, _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe, _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _ypy.si, segments_2]}]> but was:<[{indexVersion=1406477990053,generation=2,filelist=[_bta.fdt, _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim, _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _nik.si, _yok.cfe, _yok.cfs, _yok.si, _yp3.cfe, _yp3.cfs, _yp3.si, _yp4.cfe, _yp4.cfs, _yp4.si, _yp5.cfe, _yp5.cfs, _yp5.si, _yp6.cfe, _yp6.cfs, _yp6.si, _yp7.cfe, _yp7.cfs, _yp7.si, _yp8.cfe, _yp8.cfs, _yp8.si, _yp9.cfe, _yp9.cfs, _yp9.si, _ypa.cfe, _ypa.cfs, _ypa.si, _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe, _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _ypy.si, segments_2]}, {indexVersion=1406477990053,generation=3,filelist=[_bta.fdt, _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim, _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _nik.si, _ypc.cfe, _ypc.cfs, _ypc.si, _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe, _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _ypy.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1406477990053,generation=2,filelist=[_bta.fdt, _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim, _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _nik.si, _yok.cfe, _yok.cfs, _yok.si, _yp3.cfe, _yp3.cfs, _yp3.si, _yp4.cfe, _yp4.cfs, _yp4.si, _yp5.cfe, _yp5.cfs, _yp5.si, _yp6.cfe, _yp6.cfs, _yp6.si, _yp7.cfe, _yp7.cfs, _yp7.si, _yp8.cfe, _yp8.cfs, _yp8.si, _yp9.cfe, _yp9.cfs, _yp9.si, _ypa.cfe, _ypa.cfs, _ypa.si, _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe, _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _ypy.si, segments_2]}]> but was:<[{indexVersion=1406477990053,generation=2,filelist=[_bta.fdt, _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim, _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _nik.si, _yok.cfe, _yok.cfs, _yok.si, _yp3.cfe, _yp3.cfs, _yp3.si, _yp4.cfe, _yp4.cfs, _yp4.si, _yp5.cfe, _yp5.cfs, _yp5.si, _yp6.cfe, _yp6.cfs, _yp6.si, _yp7.cfe, _yp7.cfs, _yp7.si, _yp8.cfe, _yp8.cfs, _yp8.si, _yp9.cfe, _yp9.cfs, _yp9.si, _ypa.cfe, _ypa.cfs, _ypa.si, _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe, _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _ypy.si, segments_2]}, {indexVersion=1406477990053,generation=3,filelist=[_bta.fdt, _bta.fdx, _bta.fnm, _bta.si, _bta_Lucene41_0.doc, _bta_Lucene41_0.tim, _bta_Lucene41_0.tip, _bta_nrm.cfe, _bta_nrm.cfs, _nik.cfe, _nik.cfs, _nik.si, _ypc.cfe, _ypc.cfs, _ypc.si, _ypu.cfe, _ypu.cfs, _ypu.si, _ypv.cfe, _ypv.cfs, _ypv.si, _ypw.cfe, _ypw.cfs, _ypw.si, _ypx.cfe, _ypx.cfs, _ypx.si, _ypy.cfe, _ypy.cfs, _ypy.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC:C128D6FAFE8166BF]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1190)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandlerBackup

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup:     1) Thread[id=31922, name=Thread-8018, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]         at java.net.PlainSocketImpl.socketConnect(Native Method)         at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)         at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)         at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)         at java.net.Socket.connect(Socket.java:579)         at java.net.Socket.connect(Socket.java:528)         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)         at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)         at java.net.URL.openStream(URL.java:1037)         at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup: 
   1) Thread[id=31922, name=Thread-8018, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
        at java.net.PlainSocketImpl.socketConnect(Native Method)
        at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
        at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
        at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
        at java.net.Socket.connect(Socket.java:579)
        at java.net.Socket.connect(Socket.java:528)
        at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
        at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
        at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
        at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
        at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
        at java.net.URL.openStream(URL.java:1037)
        at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
	at __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandlerBackup

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=31922, name=Thread-8018, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]         at java.net.PlainSocketImpl.socketConnect(Native Method)         at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)         at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)         at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)         at java.net.Socket.connect(Socket.java:579)         at java.net.Socket.connect(Socket.java:528)         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)         at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)         at java.net.URL.openStream(URL.java:1037)         at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=31922, name=Thread-8018, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
        at java.net.PlainSocketImpl.socketConnect(Native Method)
        at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
        at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
        at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
        at java.net.Socket.connect(Socket.java:579)
        at java.net.Socket.connect(Socket.java:528)
        at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
        at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
        at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
        at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
        at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
        at java.net.URL.openStream(URL.java:1037)
        at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
	at __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC]:0)




Build Log:
[...truncated 12533 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/init-core-data-001
   [junit4]   2> 4385109 T16350 oas.SolrTestCaseJ4.setUp ###Starting doTestDetails
   [junit4]   2> 4385124 T16350 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 4385130 T16350 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36483
   [junit4]   2> 4385130 T16350 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 4385131 T16350 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4385131 T16350 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001
   [junit4]   2> 4385131 T16350 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/'
   [junit4]   2> 4385175 T16350 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/solr.xml
   [junit4]   2> 4385253 T16350 oasc.CoreContainer.<init> New CoreContainer 248555160
   [junit4]   2> 4385254 T16350 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/]
   [junit4]   2> 4385255 T16350 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4385256 T16350 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 4385256 T16350 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4385256 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4385257 T16350 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4385257 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4385257 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4385258 T16350 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4385258 T16350 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4385259 T16350 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 4385259 T16350 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4385259 T16350 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4385260 T16350 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 4385265 T16360 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/'
   [junit4]   2> 4385314 T16360 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 4385356 T16360 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4385357 T16360 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 4385363 T16360 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 4385380 T16360 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4385381 T16360 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/
   [junit4]   2> 4385382 T16360 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4385382 T16360 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/
   [junit4]   2> 4385382 T16360 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 4385383 T16360 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data
   [junit4]   2> 4385383 T16360 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index/
   [junit4]   2> 4385383 T16360 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4385384 T16360 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index
   [junit4]   2> 4385384 T16360 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=67.6337890625, floorSegmentMB=1.693359375, forceMergeDeletesPctAllowed=19.688060269334418, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5920792516815081
   [junit4]   2> 4385385 T16360 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_1,generation=1}
   [junit4]   2> 4385385 T16360 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4385386 T16360 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4385386 T16360 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4385387 T16360 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4385387 T16360 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4385387 T16360 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4385387 T16360 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4385388 T16360 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4385406 T16360 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4385409 T16360 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4385409 T16360 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4385409 T16360 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=9, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8239541096061591]
   [junit4]   2> 4385410 T16360 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_1,generation=1}
   [junit4]   2> 4385410 T16360 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4385410 T16360 oass.SolrIndexSearcher.<init> Opening Searcher@76f2320d[collection1] main
   [junit4]   2> 4385411 T16360 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
   [junit4]   2> 4385411 T16360 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf}
   [junit4]   2> 4385411 T16360 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
   [junit4]   2> 4385412 T16360 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 4385413 T16360 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
   [junit4]   2> 4385413 T16360 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 4385413 T16360 oash.ReplicationHandler.inform Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 4385414 T16360 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 4385414 T16360 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4385414 T16361 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@76f2320d[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4385414 T16350 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 4385415 T16350 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 4385428 T16350 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 4385433 T16350 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36484
   [junit4]   2> 4385433 T16350 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 4385434 T16350 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4385434 T16350 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002
   [junit4]   2> 4385434 T16350 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/'
   [junit4]   2> 4385451 T16350 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/solr.xml
   [junit4]   2> 4385495 T16350 oasc.CoreContainer.<init> New CoreContainer 1601195805
   [junit4]   2> 4385495 T16350 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/]
   [junit4]   2> 4385497 T16350 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4385497 T16350 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 4385497 T16350 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4385498 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4385498 T16350 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4385498 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4385499 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4385499 T16350 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4385499 T16350 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4385500 T16350 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 4385500 T16350 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4385501 T16350 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4385501 T16350 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 4385506 T16372 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/'
   [junit4]   2> 4385555 T16372 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 4385609 T16372 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4385610 T16372 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 4385616 T16372 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 4385628 T16372 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4385629 T16372 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/
   [junit4]   2> 4385630 T16372 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4385630 T16372 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/
   [junit4]   2> 4385630 T16372 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 4385631 T16372 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data
   [junit4]   2> 4385631 T16372 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index/
   [junit4]   2> 4385631 T16372 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4385633 T16372 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index
   [junit4]   2> 4385633 T16372 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=67.6337890625, floorSegmentMB=1.693359375, forceMergeDeletesPctAllowed=19.688060269334418, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5920792516815081
   [junit4]   2> 4385634 T16372 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33),segFN=segments_1,generation=1}
   [junit4]   2> 4385635 T16372 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4385635 T16372 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4385636 T16372 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4385636 T16372 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4385636 T16372 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4385636 T16372 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4385637 T16372 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4385637 T16372 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4385639 T16372 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4385641 T16372 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4385641 T16372 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4385642 T16372 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=9, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8239541096061591]
   [junit4]   2> 4385642 T16372 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33),segFN=segments_1,generation=1}
   [junit4]   2> 4385643 T16372 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4385643 T16372 oass.SolrIndexSearcher.<init> Opening Searcher@542a0e5a[collection1] main
   [junit4]   2> 4385643 T16372 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/conf
   [junit4]   2> 4385644 T16372 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/conf}
   [junit4]   2> 4385644 T16372 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/conf
   [junit4]   2> 4385645 T16372 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 4385645 T16372 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/conf
   [junit4]   2> 4385646 T16372 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 4385646 T16372 oash.SnapPuller.startExecutorService Poll Scheduled at an interval of 1000ms
   [junit4]   2> 4385647 T16372 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4385647 T16373 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@542a0e5a[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4385647 T16350 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 4385648 T16350 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C9906 name=collection1 org.apache.solr.core.SolrCore@3ca433db
   [junit4]   2> 4385651 T16353 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&version=2&q=*:*&wt=javabin} hits=0 status=0 QTime=1 
   [junit4]   2> 4385654 T16353 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385659 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&slave=false&command=details&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> ASYNC  NEW_CORE C9907 name=collection1 org.apache.solr.core.SolrCore@1966442d
   [junit4]   2> 4385660 T16365 C9907 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin} status=0 QTime=4 
   [junit4]   2> 4385663 T16353 C9906 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[0]} 0 0
   [junit4]   2> 4385666 T16353 C9906 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 4385667 T16353 C9906 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_2,generation=2}
   [junit4]   2> 4385669 T16353 C9906 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 4385671 T16353 C9906 oass.SolrIndexSearcher.<init> Opening Searcher@422aca31[collection1] main
   [junit4]   2> 4385671 T16353 C9906 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 4385671 T16361 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@422aca31[collection1] main{StandardDirectoryReader(segments_2:3:nrt _0(4.10):c1)}
   [junit4]   2> 4385672 T16353 C9906 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={waitSearcher=true&version=2&softCommit=false&commit=true&wt=javabin} {commit=} 0 6
   [junit4]   2> 4385721 T16376 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 4385723 T16376 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 4385726 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385727 T16376 oash.SnapPuller.fetchLatestIndex Master's generation: 2
   [junit4]   2> 4385727 T16376 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
   [junit4]   2> 4385727 T16376 oash.SnapPuller.fetchLatestIndex Starting replication process
   [junit4]   2> 4385732 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filelist&version=2&generation=2&wt=javabin} status=0 QTime=2 
   [junit4]   2> 4385732 T16376 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 4
   [junit4]   2> 4385733 T16376 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919782
   [junit4]   2> 4385733 T16376 oasu.DefaultSolrCoreState.closeIndexWriter Closing IndexWriter...
   [junit4]   2> 4385733 T16376 oasu.DefaultSolrCoreState.closeIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 4385734 T16376 oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter... core=collection1
   [junit4]   2> 4385734 T16376 oash.SnapPuller.fetchLatestIndex Starting download to MockDirectoryWrapper(RAMDirectory@27b3a72 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6f171dfe) fullCopy=false
   [junit4]   2> 4385737 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=2&wt=filestream&file=_0.cfe} status=0 QTime=0 
   [junit4]   2> 4385741 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=2&wt=filestream&file=_0.si} status=0 QTime=0 
   [junit4]   2> 4385744 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=2&wt=filestream&file=segments_2} status=0 QTime=0 
   [junit4]   2> 4385748 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=2&wt=filestream&file=_0.cfs} status=0 QTime=0 
   [junit4]   2> 4385748 T16376 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
   [junit4]   2> 4385750 T16376 oasu.DefaultSolrCoreState.openIndexWriter Creating new IndexWriter...
   [junit4]   2> 4385750 T16376 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@191825
   [junit4]   2> 4385751 T16376 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33),segFN=segments_2,generation=2}
   [junit4]   2> 4385753 T16376 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 4385753 T16376 oasu.DefaultSolrCoreState.openIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 4385754 T16376 oass.SolrIndexSearcher.<init> Opening Searcher@433a3fe2[collection1] main
   [junit4]   2> 4385754 T16373 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@433a3fe2[collection1] main{StandardDirectoryReader(segments_2:3:nrt _0(4.10):c1)}
   [junit4]   2> 4385755 T16376 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919782 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919782;done=true>>]
   [junit4]   2> 4385755 T16376 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919782
   [junit4]   2> 4385756 T16366 C9907 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={masterUrl=http://127.0.0.1:36483/solr/replication&command=fetchindex&wait=true} status=0 QTime=35 
   [junit4]   2> 4385766 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&slave=false&command=details&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385766 T16365 C9907 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin} status=0 QTime=2 
   [junit4]   2> 4385769 T16353 C9906 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[1]} 0 0
   [junit4]   2> 4385773 T16353 C9906 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 4385775 T16353 C9906 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_3,generation=3}
   [junit4]   2> 4385775 T16353 C9906 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 4385776 T16353 C9906 oass.SolrIndexSearcher.<init> Opening Searcher@5c29df30[collection1] main
   [junit4]   2> 4385776 T16353 C9906 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 4385776 T16361 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5c29df30[collection1] main{StandardDirectoryReader(segments_3:5:nrt _0(4.10):c1 _1(4.10):c1)}
   [junit4]   2> 4385777 T16353 C9906 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={waitSearcher=true&version=2&softCommit=false&commit=true&wt=javabin} {commit=} 0 4
   [junit4]   2> 4385778 T16379 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 4385779 T16379 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 4385781 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385781 T16379 oash.SnapPuller.fetchLatestIndex Master's generation: 3
   [junit4]   2> 4385781 T16379 oash.SnapPuller.fetchLatestIndex Slave's generation: 2
   [junit4]   2> 4385782 T16379 oash.SnapPuller.fetchLatestIndex Starting replication process
   [junit4]   2> 4385784 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filelist&version=2&generation=3&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385785 T16379 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 7
   [junit4]   2> 4385785 T16379 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919834
   [junit4]   2> 4385787 T16379 oasu.DefaultSolrCoreState.closeIndexWriter Closing IndexWriter...
   [junit4]   2> 4385787 T16379 oasu.DefaultSolrCoreState.closeIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 4385788 T16379 oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter... core=collection1
   [junit4]   2> 4385789 T16379 oash.SnapPuller.fetchLatestIndex Starting download to MockDirectoryWrapper(RAMDirectory@12661766 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@16d95102) fullCopy=false
   [junit4]   2> 4385789 T16379 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfe because it already exists
   [junit4]   2> 4385792 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=3&wt=filestream&file=_1.cfs} status=0 QTime=0 
   [junit4]   2> 4385796 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=3&wt=filestream&file=_1.cfe} status=0 QTime=0 
   [junit4]   2> 4385799 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=3&wt=filestream&file=segments_3} status=0 QTime=0 
   [junit4]   2> 4385799 T16379 oash.SnapPuller.downloadIndexFiles Skipping download for _0.si because it already exists
   [junit4]   2> 4385800 T16379 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfs because it already exists
   [junit4]   2> 4385802 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=3&wt=filestream&file=_1.si} status=0 QTime=0 
   [junit4]   2> 4385802 T16379 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
   [junit4]   2> 4385804 T16379 oasu.DefaultSolrCoreState.openIndexWriter Creating new IndexWriter...
   [junit4]   2> 4385804 T16379 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@780b21e4
   [junit4]   2> 4385805 T16379 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33),segFN=segments_3,generation=3}
   [junit4]   2> 4385806 T16379 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 4385806 T16379 oasu.DefaultSolrCoreState.openIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 4385807 T16379 oass.SolrIndexSearcher.<init> Opening Searcher@54f1656a[collection1] main
   [junit4]   2> 4385807 T16373 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54f1656a[collection1] main{StandardDirectoryReader(segments_3:5:nrt _0(4.10):c1 _1(4.10):c1)}
   [junit4]   2> 4385808 T16379 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919834 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919834;done=true>>]
   [junit4]   2> 4385808 T16379 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919834
   [junit4]   2> 4385808 T16366 C9907 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={masterUrl=http://127.0.0.1:36483/solr/replication&command=fetchindex&wait=true} status=0 QTime=30 
   [junit4]   2> 4385813 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&slave=false&command=details&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385814 T16365 C9907 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin} status=0 QTime=3 
   [junit4]   2> 4385816 T16353 C9906 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[2]} 0 0
   [junit4]   2> 4385819 T16353 C9906 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 4385821 T16353 C9906 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=3
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e402ab9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e4a010d),segFN=segments_4,generation=4}
   [junit4]   2> 4385821 T16353 C9906 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 4385822 T16353 C9906 oass.SolrIndexSearcher.<init> Opening Searcher@10e0c2[collection1] main
   [junit4]   2> 4385822 T16353 C9906 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 4385822 T16361 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@10e0c2[collection1] main{StandardDirectoryReader(segments_4:7:nrt _0(4.10):c1 _1(4.10):c1 _2(4.10):c1)}
   [junit4]   2> 4385823 T16353 C9906 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={waitSearcher=true&version=2&softCommit=false&commit=true&wt=javabin} {commit=} 0 4
   [junit4]   2> 4385824 T16381 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 4385825 T16381 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 4385827 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385827 T16381 oash.SnapPuller.fetchLatestIndex Master's generation: 4
   [junit4]   2> 4385827 T16381 oash.SnapPuller.fetchLatestIndex Slave's generation: 3
   [junit4]   2> 4385828 T16381 oash.SnapPuller.fetchLatestIndex Starting replication process
   [junit4]   2> 4385831 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filelist&version=2&generation=4&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385832 T16381 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 10
   [junit4]   2> 4385832 T16381 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919881
   [junit4]   2> 4385833 T16381 oasu.DefaultSolrCoreState.closeIndexWriter Closing IndexWriter...
   [junit4]   2> 4385833 T16381 oasu.DefaultSolrCoreState.closeIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 4385833 T16381 oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter... core=collection1
   [junit4]   2> 4385835 T16381 oash.SnapPuller.fetchLatestIndex Starting download to MockDirectoryWrapper(RAMDirectory@500b9d3b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3390947e) fullCopy=false
   [junit4]   2> 4385838 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=4&wt=filestream&file=_2.si} status=0 QTime=0 
   [junit4]   2> 4385839 T16381 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfe because it already exists
   [junit4]   2> 4385839 T16381 oash.SnapPuller.downloadIndexFiles Skipping download for _1.cfs because it already exists
   [junit4]   2> 4385839 T16381 oash.SnapPuller.downloadIndexFiles Skipping download for _1.cfe because it already exists
   [junit4]   2> 4385842 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=4&wt=filestream&file=_2.cfe} status=0 QTime=0 
   [junit4]   2> 4385845 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=4&wt=filestream&file=segments_4} status=0 QTime=0 
   [junit4]   2> 4385845 T16381 oash.SnapPuller.downloadIndexFiles Skipping download for _0.si because it already exists
   [junit4]   2> 4385846 T16381 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfs because it already exists
   [junit4]   2> 4385848 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=filecontent&checksum=true&generation=4&wt=filestream&file=_2.cfs} status=0 QTime=0 
   [junit4]   2> 4385900 T16381 oash.SnapPuller.downloadIndexFiles Skipping download for _1.si because it already exists
   [junit4]   2> 4385901 T16381 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
   [junit4]   2> 4385902 T16381 oasu.DefaultSolrCoreState.openIndexWriter Creating new IndexWriter...
   [junit4]   2> 4385903 T16381 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3c84c37a
   [junit4]   2> 4385904 T16381 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76d9fe9d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71e35a33),segFN=segments_4,generation=4}
   [junit4]   2> 4385904 T16381 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 4385904 T16381 oasu.DefaultSolrCoreState.openIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 4385906 T16381 oass.SolrIndexSearcher.<init> Opening Searcher@1036b53[collection1] main
   [junit4]   2> 4385906 T16373 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1036b53[collection1] main{StandardDirectoryReader(segments_4:7:nrt _0(4.10):c1 _1(4.10):c1 _2(4.10):c1)}
   [junit4]   2> 4385906 T16381 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919881 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919881;done=true>>]
   [junit4]   2> 4385907 T16381 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index.20140727120919881
   [junit4]   2> 4385907 T16366 C9907 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={masterUrl=http://127.0.0.1:36483/solr/replication&command=fetchindex&wait=true} status=0 QTime=83 
   [junit4]   2> 4385923 T16350 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 4385929 T16350 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36489
   [junit4]   2> 4385930 T16350 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 4385930 T16350 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4385930 T16350 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003
   [junit4]   2> 4385931 T16350 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/'
   [junit4]   2> 4385955 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4385956 T16374 oash.SnapPuller.fetchLatestIndex Slave in sync with master.
   [junit4]   2> 4385977 T16350 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/solr.xml
   [junit4]   2> 4386058 T16350 oasc.CoreContainer.<init> New CoreContainer 1726482995
   [junit4]   2> 4386058 T16350 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/]
   [junit4]   2> 4386060 T16350 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4386060 T16350 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 4386060 T16350 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4386061 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4386061 T16350 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4386061 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4386062 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4386062 T16350 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4386062 T16350 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4386063 T16350 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 4386063 T16350 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4386064 T16350 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4386064 T16350 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 4386069 T16392 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/'
   [junit4]   2> 4386120 T16392 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 4386162 T16392 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4386163 T16392 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 4386169 T16392 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 4386186 T16392 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4386187 T16392 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/
   [junit4]   2> 4386188 T16392 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4386188 T16392 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/
   [junit4]   2> 4386188 T16392 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 4386189 T16392 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data
   [junit4]   2> 4386189 T16392 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index/
   [junit4]   2> 4386189 T16392 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4386190 T16392 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index
   [junit4]   2> 4386190 T16392 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=67.6337890625, floorSegmentMB=1.693359375, forceMergeDeletesPctAllowed=19.688060269334418, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5920792516815081
   [junit4]   2> 4386191 T16392 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e0e6a08 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@be33180),segFN=segments_1,generation=1}
   [junit4]   2> 4386191 T16392 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4386192 T16392 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4386193 T16392 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4386193 T16392 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4386193 T16392 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4386193 T16392 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4386193 T16392 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4386194 T16392 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4386212 T16392 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4386214 T16392 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4386215 T16392 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4386215 T16392 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=9, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8239541096061591]
   [junit4]   2> 4386216 T16392 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e0e6a08 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@be33180),segFN=segments_1,generation=1}
   [junit4]   2> 4386216 T16392 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4386216 T16392 oass.SolrIndexSearcher.<init> Opening Searcher@b696276[collection1] main
   [junit4]   2> 4386217 T16392 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/conf
   [junit4]   2> 4386217 T16392 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/conf}
   [junit4]   2> 4386217 T16392 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/conf
   [junit4]   2> 4386218 T16392 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 4386218 T16392 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/conf
   [junit4]   2> 4386219 T16392 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 4386219 T16392 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 4386219 T16392 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 4386220 T16392 oash.ReplicationHandler.inform Replication enabled for following config files: schema.xml
   [junit4]   2> 4386220 T16392 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 4386220 T16392 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4386220 T16393 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@b696276[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4386221 T16350 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 4386221 T16350 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 4386227 T16355 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&slave=false&command=details&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> ASYNC  NEW_CORE C9908 name=collection1 org.apache.solr.core.SolrCore@698ce178
   [junit4]   2> 4386227 T16385 C9908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&qt=/replication&command=details&version=2&wt=javabin} status=0 QTime=4 
   [junit4]   2> 4386228 T16350 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1726482995
   [junit4]   2> 4386230 T16350 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@698ce178
   [junit4]   2> 4386230 T16350 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 4386231 T16350 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 4386231 T16350 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 4386232 T16350 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 4386232 T16350 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 4386233 T16350 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 4386233 T16350 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data;done=false>>]
   [junit4]   2> 4386234 T16350 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data
   [junit4]   2> 4386234 T16350 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index;done=false>>]
   [junit4]   2> 4386234 T16350 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-003/collection1/data/index
   [junit4]   2> 4386235 T16350 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 4386954 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4386955 T16374 oash.SnapPuller.fetchLatestIndex Slave in sync with master.
   [junit4]   2> 4387955 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4387955 T16374 oash.SnapPuller.fetchLatestIndex Slave in sync with master.
   [junit4]   2> 4388956 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4388956 T16374 oash.SnapPuller.fetchLatestIndex Slave in sync with master.
   [junit4]   2> 4389955 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4389956 T16374 oash.SnapPuller.fetchLatestIndex Slave in sync with master.
   [junit4]   2> 4390956 T16354 C9906 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 4390956 T16374 oash.SnapPuller.fetchLatestIndex Slave in sync with master.
   [junit4]   2> 4391238 T16350 oas.SolrTestCaseJ4.tearDown ###Ending doTestDetails
   [junit4]   2> 4391238 T16350 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=248555160
   [junit4]   2> 4391241 T16350 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3ca433db
   [junit4]   2> 4391241 T16350 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=3,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 4391244 T16350 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 4391245 T16350 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 4391245 T16350 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 4391248 T16350 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 4391249 T16350 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 4391249 T16350 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 4391249 T16350 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data
   [junit4]   2> 4391250 T16350 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index;done=false>>]
   [junit4]   2> 4391250 T16350 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index
   [junit4]   2> 4391251 T16350 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 4396245 T16350 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1601195805
   [junit4]   2> 4396247 T16350 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1966442d
   [junit4]   2> 4406954 T16374 oash.SnapPuller.fetchLatestIndex ERROR Master at: http://127.0.0.1:36483/solr is not available. Index fetch failed. Exception: IOException occured when talking to server at: http://127.0.0.1:36483/solr
   [junit4]   2> 4406957 T16350 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 4406957 T16350 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 4406958 T16350 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 4406958 T16350 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 4406960 T16350 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 4406961 T16350 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 4406961 T16350 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data;done=false>>]
   [junit4]   2> 4406962 T16350 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data
   [junit4]   2> 4406962 T16350 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index;done=false>>]
   [junit4]   2> 4406962 T16350 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-002/collection1/data/index
   [junit4]   2> 4406963 T16350 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 4411271 T16350 oas.SolrTestCaseJ4.setUp ###Starting doTestReplicateAfterStartupWithNoActivity
   [junit4]   2> 4411286 T16350 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 4411291 T16350 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36495
   [junit4]   2> 4411292 T16350 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 4411292 T16350 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4411292 T16350 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001
   [junit4]   2> 4411293 T16350 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/'
   [junit4]   2> 4411336 T16350 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/solr.xml
   [junit4]   2> 4411429 T16350 oasc.CoreContainer.<init> New CoreContainer 1715233452
   [junit4]   2> 4411430 T16350 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/]
   [junit4]   2> 4411431 T16350 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4411431 T16350 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 4411432 T16350 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4411432 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4411432 T16350 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4411433 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4411433 T16350 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4411433 T16350 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4411434 T16350 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4411434 T16350 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 4411435 T16350 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4411435 T16350 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4411435 T16350 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 4411440 T16404 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/'
   [junit4]   2> 4411490 T16404 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 4411532 T16404 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4411532 T16404 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 4411539 T16404 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 4411556 T16404 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4411557 T16404 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/
   [junit4]   2> 4411558 T16404 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4411558 T16404 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/
   [junit4]   2> 4411558 T16404 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 4411559 T16404 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data
   [junit4]   2> 4411559 T16404 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index/
   [junit4]   2> 4411559 T16404 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4411560 T16404 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/data/index
   [junit4]   2> 4411560 T16404 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=67.6337890625, floorSegmentMB=1.693359375, forceMergeDeletesPctAllowed=19.688060269334418, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5920792516815081
   [junit4]   2> 4411561 T16404 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@794627d4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4368d6a4),segFN=segments_1,generation=1}
   [junit4]   2> 4411561 T16404 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4411562 T16404 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4411562 T16404 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4411563 T16404 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4411563 T16404 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4411563 T16404 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4411563 T16404 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4411564 T16404 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4411582 T16404 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4411585 T16404 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4411585 T16404 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4411585 T16404 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=9, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8239541096061591]
   [junit4]   2> 4411586 T16404 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@794627d4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4368d6a4),segFN=segments_1,generation=1}
   [junit4]   2> 4411586 T16404 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4411586 T16404 oass.SolrIndexSearcher.<init> Opening Searcher@c13b75b[collection1] main
   [junit4]   2> 4411587 T16404 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
   [junit4]   2> 4411587 T16404 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf}
   [junit4]   2> 4411587 T16404 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
   [junit4]   2> 4411588 T16404 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 4411589 T16404 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.handler.TestReplicationHandler-E4FFCDCA8EC968BC-001/solr-instance-001/collection1/conf
   [junit4]   2> 4411589 T16404 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 4411589 T16404 oash.ReplicationHandler.inform Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 4411589 T16404 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 4411590 T16404 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4411590 T16405 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@c13b75b[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4411590 T16350 oas

[...truncated too long message...]

ieldUpdateProcessorsTest, TestMiniSolrCloudCluster, TestBlendedInfixSuggestions, ResponseLogComponentTest, SpellPossibilityIteratorTest, AnalyticsQueryTest, BlockCacheTest, OverseerRolesTest, TestCodecSupport, ReplicationFactorTest, ClusterStateUpdateTest, DocValuesMultiTest, EnumFieldTest, TestDistributedSearch, UnloadDistributedZkTest, CoreMergeIndexesAdminHandlerTest, TestLMJelinekMercerSimilarityFactory, ExternalFileFieldSortTest, TestFastOutputStream, InfoHandlerTest, TestSort, TestManagedSchema, TestDocBasedVersionConstraints, ShardSplitTest, TestStressUserVersions, TestShardHandlerFactory, TestDistribDocBasedVersion, CachingDirectoryFactoryTest, SuggesterTest, PingRequestHandlerTest, TestCollationFieldDocValues, TestFuzzyAnalyzedSuggestions, DistributedExpandComponentTest, CustomCollectionTest, DeleteInactiveReplicaTest, TestManagedSchemaFieldResource, TestFreeTextSuggestions, DeleteReplicaTest, TestHighFrequencyDictionaryFactory, ReturnFieldsTest, HdfsBasicDistributedZkTest, TestSolrQueryParserDefaultOperatorResource, TestRangeQuery, TestCopyFieldCollectionResource, TestSurroundQueryParser, WordBreakSolrSpellCheckerTest, SpellCheckCollatorTest, DocValuesMissingTest, TestSearchPerf, TestStressRecovery, TestReload, TestConfig, BasicZkTest, LeaderElectionTest, ZkSolrClientTest, TestZkChroot, ZkCLITest, ShardRoutingCustomTest, AutoCommitTest, TestDistributedGrouping, TestFaceting, TestHashPartitioner, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, ZkControllerTest, TestStressReorder, TestMultiCoreConfBootstrap, TestStressVersions, HardAutoCommitTest, SimpleFacetsTest, TestSolr4Spatial, SpellCheckComponentTest, TestGroupingSearch, PeerSyncTest, BadIndexSchemaTest, TestFunctionQuery, BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, TestBadConfig, OverseerCollectionProcessorTest, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SuggesterFSTTest, CoreAdminHandlerTest, SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTSTTest, SuggesterWFSTTest, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, QueryEqualityTest, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, TestSolrDeletionPolicy1, DebugComponentTest, DocumentAnalysisRequestHandlerTest, TestQueryTypes, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, MoreLikeThisHandlerTest, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, LoggingHandlerTest, SolrPluginUtilsTest, IndexSchemaTest, TestJmxIntegration, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestComponentsName, CoreContainerCoreInitFailuresTest, TestBinaryResponseWriter, TestLFUCache, HighlighterConfigTest, SOLR749Test, TestQuerySenderListener, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, SolrIndexConfigTest, BadComponentTest, TestSolrDeletionPolicy2, MultiTermTest, TestDocSet, TestBinaryField, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, TestSolrCoreProperties, TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, TestCharFilters, SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest, TestDFRSimilarityFactory, TestNumberUtils, TestIBSimilarityFactory, TestBM25SimilarityFactory, TestDefaultSimilarityFactory, TestFastWriter, LegacyHTMLStripCharFilterTest, TestJmxMonitoredMap, TimeZoneUtilsTest, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, TestSolrXMLSerializer, TestSystemIdResolver, TestSuggestSpellingConverter, DateFieldTest, RAMDirectoryFactoryTest, TestSolrJ, TestUtils, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestDistributedMissingSort, TestTolerantSearch, AliasIntegrationTest, ConnectionManagerTest, DeleteShardTest, DistribCursorPagingTest, HttpPartitionTest, MigrateRouteKeyTest, TestShortCircuitedRequests, HdfsChaosMonkeySafeLeaderTest, HdfsRecoveryZkTest, HdfsSyncSliceTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, TestInfoStreamLogging, TestNRTOpen, TestReloadAndDeleteDocs, TestSolrXml, TestSolrXmlPersistence, TestReplicationHandlerBackup]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestReplicationHandlerBackup -Dtests.seed=E4FFCDCA8EC968BC -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=sr_BA_#Latn -Dtests.timezone=America/Edmonton -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | TestReplicationHandlerBackup (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup: 
   [junit4]    >    1) Thread[id=31922, name=Thread-8018, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
   [junit4]    >         at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
   [junit4]    >         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]    >         at java.net.Socket.connect(Socket.java:579)
   [junit4]    >         at java.net.Socket.connect(Socket.java:528)
   [junit4]    >         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
   [junit4]    >         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
   [junit4]    >         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
   [junit4]    >         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
   [junit4]    >         at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
   [junit4]    >         at java.net.URL.openStream(URL.java:1037)
   [junit4]    >         at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=31922, name=Thread-8018, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
   [junit4]    >         at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
   [junit4]    >         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]    >         at java.net.Socket.connect(Socket.java:579)
   [junit4]    >         at java.net.Socket.connect(Socket.java:528)
   [junit4]    >         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
   [junit4]    >         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
   [junit4]    >         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
   [junit4]    >         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
   [junit4]    >         at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
   [junit4]    >         at java.net.URL.openStream(URL.java:1037)
   [junit4]    >         at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E4FFCDCA8EC968BC]:0)
   [junit4] Completed on J0 in 30.12s, 1 test, 2 errors <<< FAILURES!

[...truncated 135 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 410 suites, 1702 tests, 2 suite-level errors, 1 failure, 136 ignored (8 assumptions)

Total time: 246 minutes 18 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #582
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 687356687 bytes
Compression is 0.0%
Took 1 min 48 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 584 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/584/

1 tests failed.
FAILED:  org.apache.solr.cloud.OverseerTest.testOverseerFailure

Error Message:
Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed

Stack Trace:
org.apache.solr.common.SolrException: Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
	at __randomizedtesting.SeedInfo.seed([3C225866AD0181FE:382AD795BFA46EDF]:0)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:144)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:163)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:125)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:155)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:314)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:221)
	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:155)
	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:660)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:40)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:131)
	... 48 more
Caused by: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:432)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:429)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:386)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:373)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:136)
	at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:34)
	... 49 more




Build Log:
[...truncated 12050 lines...]
   [junit4] Suite: org.apache.solr.cloud.OverseerTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.OverseerTest-3C225866AD0181FE-001/init-core-data-001
   [junit4]   2> 3874968 T13946 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 3874968 T13946 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 3874968 T13946 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 3874973 T13946 oas.SolrTestCaseJ4.setUp ###Starting testDoubleAssignment
   [junit4]   2> 3874974 T13946 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3874974 T13947 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 3875074 T13946 oasc.ZkTestServer.run start zk server on port:37743
   [junit4]   2> 3875075 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3875079 T13953 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b1e31df name:ZooKeeperConnection Watcher:127.0.0.1:37743/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3875079 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3875080 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3875081 T13955 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3505ce93 name:ZooKeeperConnection Watcher:127.0.0.1:37743 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3875081 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3875083 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3875084 T13957 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66b4c57c name:ZooKeeperConnection Watcher:127.0.0.1:37743 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3875084 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3875085 T13946 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 3875087 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 3875089 T13946 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 3875090 T13946 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 3875091 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3875093 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3875093 T13959 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f000156 name:ZooKeeperConnection Watcher:127.0.0.1:37743/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3875094 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3875095 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3875096 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 3875098 T13959 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3875098 T13953 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3875098 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3875099 T13961 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48801d38 name:ZooKeeperConnection Watcher:127.0.0.1:37743/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3875099 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3875100 T13946 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 3875101 T13946 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 3875103 T13946 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:37743_solr
   [junit4]   2> 3875103 T13946 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 3875105 T13946 oasc.Overseer.start Overseer (id=92164795816673284-127.0.0.1:37743_solr-n_0000000000) starting
   [junit4]   2> 3875106 T13946 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 3875107 T13946 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 3875108 T13946 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 3875110 T13946 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 3875111 T13946 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 3875113 T13946 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 3875116 T13963 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 3875116 T13962 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 3875117 T13962 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3875117 T13962 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3875118 T13962 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 3875118 T13962 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 3875119 T13961 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3875119 T13953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3875119 T13959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3875618 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3875623 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3875626 T13961 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3875626 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3875626 T13962 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3875628 T13959 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 3875628 T13953 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 3875629 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3875629 T13961 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3875631 T13965 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21473e3d name:ZooKeeperConnection Watcher:127.0.0.1:37743/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3875631 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3875632 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3875634 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 3875635 T13953 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3875635 T13965 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3875636 T13961 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3875636 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3875637 T13962 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3875637 T13962 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3875638 T13961 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3875639 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3875642 T13961 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3875643 T13961 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3875745 T13965 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3875745 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3875745 T13953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3875748 T13953 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 3875748 T13965 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 3875750 T13946 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:37743 37743
   [junit4]   2> 3877018 T13947 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:37743 37743
   [junit4]   2> 3877019 T13946 oas.SolrTestCaseJ4.tearDown ###Ending testDoubleAssignment
   [junit4]   2> 3877020 T13962 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:37743_solr
   [junit4]   2> 3877021 T13966 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:277)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:316)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:85)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:288)
   [junit4]   2> 
   [junit4]   2> 3877026 T13946 oas.SolrTestCaseJ4.setUp ###Starting testShardLeaderChange
   [junit4]   2> 3877027 T13946 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3877027 T13967 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 3877127 T13946 oasc.ZkTestServer.run start zk server on port:10513
   [junit4]   2> 3877128 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3877132 T13973 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@276893f4 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3877132 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3877133 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3877134 T13975 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@55ea0941 name:ZooKeeperConnection Watcher:127.0.0.1:10513 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3877135 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3877137 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3877138 T13977 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@747b77e6 name:ZooKeeperConnection Watcher:127.0.0.1:10513 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3877138 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3877139 T13946 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 3877142 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 3877145 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3877145 T13978 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3877147 T13980 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49674d06 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3877147 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3877147 T13982 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1cf1ea90 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3877148 T13978 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3877148 T13946 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 3877149 T13978 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 3877150 T13946 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 3877150 T13978 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 3877151 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3877153 T13978 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:10513_solr
   [junit4]   2> 3877153 T13978 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 3877153 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 3877154 T13978 oasc.Overseer.start Overseer (id=92164795951218692-127.0.0.1:10513_solr-n_0000000000) starting
   [junit4]   2> 3877155 T13980 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3877156 T13946 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 3877156 T13978 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 3877158 T13978 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 3877159 T13978 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 3877161 T13978 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 3877163 T13978 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 3877165 T13978 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 3877167 T13984 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 3877167 T13978 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 3877169 T13978 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3877171 T13986 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@738439ad name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3877171 T13978 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3877172 T13984 oasc.OverseerCollectionProcessor.run ERROR Unable to prioritize overseer  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /roles.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:226)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:223)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:223)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor.prioritizeOverseerNodes(OverseerCollectionProcessor.java:416)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:260)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 3877173 T13978 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:10513_solr
   [junit4]   2> 3877173 T13978 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 3877174 T13978 oasc.Overseer.start Overseer (id=92164795951218693-127.0.0.1:10513_solr-n_0000000001) starting
   [junit4]   2> 3877180 T13988 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 3877181 T13987 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 3877182 T13987 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3877182 T13987 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3877182 T13987 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 3877183 T13987 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 3877184 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877185 T13980 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3877658 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3877664 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3877667 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877668 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877668 T13987 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3877668 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3877670 T13987 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3877670 T13987 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3877672 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877672 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3877673 T13990 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40954937 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3877673 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3877675 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3877677 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 3877679 T13990 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3877679 T13980 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3877680 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877680 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3877680 T13987 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3877681 T13987 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3877682 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877683 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3877783 T13990 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3877783 T13980 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3877885 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3877887 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877887 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877888 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877888 T13987 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3877889 T13987 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 3877889 T13987 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 3877889 T13987 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 3877891 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3877992 T13990 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3877992 T13980 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878387 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3878393 T13990 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3878393 T13980 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3878393 T13980 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3878394 T13990 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3878394 T13980 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3878393 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878397 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878397 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878398 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878398 T13987 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878400 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878501 T13990 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3878597 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878598 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3878599 T13992 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9e20655 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3878600 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3878601 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3878603 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 3878605 T13992 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3878605 T13990 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3878606 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878606 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3878606 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878607 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878607 T13987 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878607 T13987 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3878608 T13987 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3878609 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878611 T13992 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3878611 T13990 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3878612 T13992 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3878612 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3878612 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878613 T13987 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3878614 T13987 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3878614 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878616 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3878617 T13994 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45e43ff1 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3878617 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878618 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3878619 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3878620 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 3878622 T13994 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3878622 T13992 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3878623 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878623 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3878624 T13987 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3878624 T13987 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3878626 T13986 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878627 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878694 T13978 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 3878696 T13978 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3878698 T13996 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@97aa954 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3878698 T13978 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3878698 T13987 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:10513_solr
   [junit4]   2> 3878700 T13997 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:277)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:316)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:85)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:288)
   [junit4]   2> 
   [junit4]   2> 3878701 T13978 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:10513_solr
   [junit4]   2> 3878701 T13978 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 3878703 T13978 oasc.Overseer.start Overseer (id=92164795951218697-127.0.0.1:10513_solr-n_0000000002) starting
   [junit4]   2> 3878708 T13999 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 3878709 T13998 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878710 T13998 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
   [junit4]   2> 3878710 T13998 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3878711 T13998 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3878711 T13994 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878711 T13992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878713 T13998 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3878714 T13998 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3878714 T13994 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878714 T13992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878716 T13992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878716 T13994 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878717 T13998 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3878718 T13998 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3878718 T13992 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878718 T13994 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3878719 T13998 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 3878829 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878831 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878831 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3878832 T13998 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878833 T13998 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 3878833 T13998 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3878834 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878836 T13994 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3878836 T13992 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3878836 T13992 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3878837 T13994 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3878837 T13992 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3878836 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3878840 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878842 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3878943 T13994 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3879040 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879041 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3879042 T14001 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18293aa2 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3879043 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3879044 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3879046 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 3879048 T13994 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3879048 T14001 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3879049 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879049 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3879049 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879050 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879050 T13998 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879051 T13998 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879051 T13998 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879052 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879053 T14001 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3879053 T13994 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3879055 T14001 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3879055 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879055 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3879056 T13998 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879056 T13998 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879057 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879059 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879059 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3879060 T14003 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13111ed0 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3879061 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3879062 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3879063 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 3879065 T14001 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3879065 T14003 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3879066 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879066 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3879067 T13998 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879067 T13998 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879068 T13996 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879069 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879112 T13978 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 3879115 T13978 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3879116 T14005 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71db6f35 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3879116 T13978 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3879116 T13998 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:10513_solr
   [junit4]   2> 3879117 T14006 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:277)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:316)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:85)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:288)
   [junit4]   2> 
   [junit4]   2> 3879118 T13978 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:10513_solr
   [junit4]   2> 3879119 T13978 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 3879120 T13978 oasc.Overseer.start Overseer (id=92164795951218700-127.0.0.1:10513_solr-n_0000000003) starting
   [junit4]   2> 3879125 T14008 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 3879126 T14007 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879127 T14007 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
   [junit4]   2> 3879127 T14007 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879127 T14007 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879128 T14001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879128 T14003 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879129 T14007 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879130 T14007 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879130 T14001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879130 T14003 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879132 T14001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879132 T14003 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879133 T14007 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879134 T14007 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879134 T14003 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879134 T14001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879135 T14007 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 3879271 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879273 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879273 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3879274 T14007 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879274 T14007 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 3879275 T14007 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879276 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879278 T14001 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3879278 T14003 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3879278 T14001 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3879279 T14003 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3879279 T14001 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3879278 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879281 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879283 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879385 T14003 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3879482 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879483 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3879484 T14010 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@54c9739a name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3879485 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3879487 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3879488 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 3879490 T14003 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3879490 T14010 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3879491 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879491 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3879492 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879492 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879492 T14007 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879493 T14007 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879493 T14007 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879495 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879496 T14010 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3879496 T14003 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3879497 T14010 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3879498 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879498 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3879499 T14007 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879499 T14007 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879500 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879502 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3879502 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879503 T14012 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c879e00 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3879503 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3879505 T13946 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3879506 T13946 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 3879508 T14010 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3879508 T14012 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3879509 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879509 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3879509 T14007 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879510 T14007 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879511 T14005 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879512 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879528 T13978 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 3879530 T13978 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3879531 T14014 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1cd5d808 name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3879532 T13978 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3879532 T14007 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:10513_solr
   [junit4]   2> 3879533 T14015 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:277)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:316)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:85)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:288)
   [junit4]   2> 
   [junit4]   2> 3879535 T13978 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:10513_solr
   [junit4]   2> 3879535 T13978 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 3879536 T13978 oasc.Overseer.start Overseer (id=92164795951218703-127.0.0.1:10513_solr-n_0000000004) starting
   [junit4]   2> 3879541 T14017 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 3879542 T14016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879543 T14016 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
   [junit4]   2> 3879543 T14016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879544 T14016 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879544 T14012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879544 T14010 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879546 T14016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879546 T14016 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879547 T14012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879547 T14010 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879548 T14010 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879548 T14012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879550 T14016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 3879550 T14016 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879551 T14010 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879551 T14012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3879551 T14016 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 3879714 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879716 T14014 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879716 T13946 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3879717 T14016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879717 T14016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 3879718 T14016 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 3879719 T14014 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879721 T14010 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3879721 T14012 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 3879721 T14010 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3879722 T14012 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3879722 T14010 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3879721 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879725 T14014 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879727 T14014 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3879828 T14012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3879925 T13946 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3879926 T13946 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3879927 T14019 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a99156b name:ZooKeeperConnection Watcher:127.0.0.1:10513/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3879928 T13946 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3879929 T13946 oascc.ZkStateReader.creat

[...truncated too long message...]

dLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:144)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:163)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:125)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leaders/shard8
   [junit4]   2> 	at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:40)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:131)
   [junit4]   2> 	... 6 more
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leaders/shard8
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:209)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:206)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:206)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:417)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:386)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:373)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:136)
   [junit4]   2> 	at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:34)
   [junit4]   2> 	... 7 more
   [junit4]   2> 
   [junit4]   2> 3918299 T14367 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3918300 T14367 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 3918300 T14367 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 3920541 T13946 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.OverseerTest-3C225866AD0181FE-001
   [junit4]   2> NOTE: test params are: codec=Lucene46, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=ar, timezone=IET
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=305604960,total=532676608
   [junit4]   2> NOTE: All tests run in this JVM: [TriLevelCompositeIdRoutingTest, TestCollectionAPI, TestDefaultSearchFieldResource, TestExceedMaxTermLength, HdfsWriteToMultipleCollectionsTest, TestCloudManagedSchemaConcurrent, TestQueryUtils, TestPostingsSolrHighlighter, TestIndexingPerformance, TestCSVLoader, TestReplicationHandlerBackup, DOMUtilTest, TestReloadAndDeleteDocs, SolrInfoMBeanTest, ChaosMonkeySafeLeaderTest, CursorMarkTest, TestDocumentBuilder, HdfsSyncSliceTest, TestSystemIdResolver, TestOmitPositions, TestCharFilters, SortByFunctionTest, IndexSchemaRuntimeFieldTest, TestFastLRUCache, TestLazyCores, RollingRestartTest, HdfsCollectionsAPIDistributedZkTest, TestStressLucene, TestFieldResource, ZkNodePropsTest, TestFunctionQuery, TermVectorComponentDistributedTest, TestIBSimilarityFactory, CacheHeaderTest, TestChildDocTransformer, TestDistributedMissingSort, TestCSVResponseWriter, TestImplicitCoreProperties, FieldAnalysisRequestHandlerTest, TestTolerantSearch, AliasIntegrationTest, TestWriterPerf, LukeRequestHandlerTest, TestFieldTypeResource, HdfsDirectoryTest, SoftAutoCommitTest, TestRandomFaceting, QueryElevationComponentTest, TestCoreContainer, SystemInfoHandlerTest, TestExpandComponent, RequestHandlersTest, BasicZkTest, TestMultiCoreConfBootstrap, ScriptEngineTest, MigrateRouteKeyTest, DebugComponentTest, TestFieldSortValues, SpellingQueryConverterTest, TestXIncludeConfig, SolrPluginUtilsTest, CopyFieldTest, TestMergePolicyConfig, DistanceFunctionTest, OutputWriterTest, TestDynamicFieldResource, TimeZoneUtilsTest, TestSuggestSpellingConverter, TestBadConfig, TestIndexSearcher, TestPseudoReturnFields, SolrCoreTest, DistributedQueryComponentCustomSortTest, StatsComponentTest, ZkSolrClientTest, TestJoin, HdfsUnloadDistributedZkTest, TestCloudManagedSchema, TestReRankQParserPlugin, SearchHandlerTest, SOLR749Test, CollectionsAPIAsyncDistributedZkTest, TestHighlightDedupGrouping, TestCollationField, BasicDistributedZk2Test, TestFiltering, LegacyHTMLStripCharFilterTest, DistributedDebugComponentTest, TestManagedSynonymFilterFactory, AutoCommitTest, DirectSolrConnectionTest, TestRestManager, ConvertedLegacyTest, TestSolrQueryParser, TestAtomicUpdateErrorCases, TestRTGBase, TestUpdate, HdfsBasicDistributedZk2Test, BasicDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest]
   [junit4] Completed on J1 in 45.61s, 9 tests, 1 error <<< FAILURES!

[...truncated 760 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 410 suites, 1702 tests, 1 error, 50 ignored (8 assumptions)

Total time: 224 minutes 15 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #582
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 640656943 bytes
Compression is 0.0%
Took 1 min 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure