You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2015/05/27 16:23:34 UTC

[JENKINS] Lucene-Solr-NightlyTests-trunk - Build # 694 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-trunk/694/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test

Error Message:
The Monkey ran for over 20 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 20 seconds and no jetties were stopped - this is worth investigating!
	at __randomizedtesting.SeedInfo.seed([BF0290866116A66E:3756AF5CCFEACB96]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:537)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest:     1) Thread[id=614, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:136)    2) Thread[id=615, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:136)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest: 
   1) Thread[id=614, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:136)
   2) Thread[id=615, name=StoppableIndexingThread, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:136)
	at __randomizedtesting.SeedInfo.seed([BF0290866116A66E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=614, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at org.apache.solr.common.util.XML.escape(XML.java:195)         at org.apache.solr.common.util.XML.escapeCharData(XML.java:77)         at org.apache.solr.common.util.XML.writeXML(XML.java:147)         at org.apache.solr.client.solrj.util.ClientUtils.writeVal(ClientUtils.java:163)         at org.apache.solr.client.solrj.util.ClientUtils.writeXML(ClientUtils.java:131)         at org.apache.solr.client.solrj.request.UpdateRequest.writeXML(UpdateRequest.java:381)         at org.apache.solr.client.solrj.request.UpdateRequest.getXML(UpdateRequest.java:297)         at org.apache.solr.client.solrj.request.RequestWriter.getContentStream(RequestWriter.java:67)         at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:95)         at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:105)         at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:342)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)         at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:152)         at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:177)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:116)    2) Thread[id=615, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]         at java.security.AccessController.getStackAccessControlContext(Native Method)         at java.security.AccessController.checkPermission(AccessController.java:858)         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)         at java.lang.reflect.AccessibleObject.setAccessible(AccessibleObject.java:95)         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.getBoundFields(ReflectiveTypeAdapterFactory.java:104)         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.create(ReflectiveTypeAdapterFactory.java:65)         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.MiniGson.getAdapter(MiniGson.java:92)         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.Gson.toJson(Gson.java:504)         at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:87)         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$4.write(SlaveMain.java:410)         at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)         at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)         at java.io.PrintStream.flush(PrintStream.java:338)         at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)         at java.io.PrintStream.write(PrintStream.java:482)         at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)         at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)         at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)         at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)         at java.io.PrintStream.write(PrintStream.java:527)         at java.io.PrintStream.print(PrintStream.java:669)         at java.io.PrintStream.println(PrintStream.java:823)         at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)         at java.lang.Throwable.printStackTrace(Throwable.java:658)         at java.lang.Throwable.printStackTrace(Throwable.java:643)         at java.lang.Throwable.printStackTrace(Throwable.java:634)         at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:122)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=614, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at org.apache.solr.common.util.XML.escape(XML.java:195)
        at org.apache.solr.common.util.XML.escapeCharData(XML.java:77)
        at org.apache.solr.common.util.XML.writeXML(XML.java:147)
        at org.apache.solr.client.solrj.util.ClientUtils.writeVal(ClientUtils.java:163)
        at org.apache.solr.client.solrj.util.ClientUtils.writeXML(ClientUtils.java:131)
        at org.apache.solr.client.solrj.request.UpdateRequest.writeXML(UpdateRequest.java:381)
        at org.apache.solr.client.solrj.request.UpdateRequest.getXML(UpdateRequest.java:297)
        at org.apache.solr.client.solrj.request.RequestWriter.getContentStream(RequestWriter.java:67)
        at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getDelegate(RequestWriter.java:95)
        at org.apache.solr.client.solrj.request.RequestWriter$LazyContentStream.getName(RequestWriter.java:105)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:342)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
        at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
        at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:152)
        at org.apache.solr.cloud.StoppableIndexingThread.indexDocs(StoppableIndexingThread.java:177)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:116)
   2) Thread[id=615, name=StoppableIndexingThread, state=RUNNABLE, group=TGRP-HdfsChaosMonkeySafeLeaderTest]
        at java.security.AccessController.getStackAccessControlContext(Native Method)
        at java.security.AccessController.checkPermission(AccessController.java:858)
        at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
        at java.lang.reflect.AccessibleObject.setAccessible(AccessibleObject.java:95)
        at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.getBoundFields(ReflectiveTypeAdapterFactory.java:104)
        at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.create(ReflectiveTypeAdapterFactory.java:65)
        at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.MiniGson.getAdapter(MiniGson.java:92)
        at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.Gson.toJson(Gson.java:504)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:87)
        at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$4.write(SlaveMain.java:410)
        at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
        at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
        at java.io.PrintStream.flush(PrintStream.java:338)
        at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
        at java.io.PrintStream.write(PrintStream.java:482)
        at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
        at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
        at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
        at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
        at java.io.PrintStream.write(PrintStream.java:527)
        at java.io.PrintStream.print(PrintStream.java:669)
        at java.io.PrintStream.println(PrintStream.java:823)
        at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
        at java.lang.Throwable.printStackTrace(Throwable.java:658)
        at java.lang.Throwable.printStackTrace(Throwable.java:643)
        at java.lang.Throwable.printStackTrace(Throwable.java:634)
        at org.apache.solr.cloud.StoppableIndexingThread.run(StoppableIndexingThread.java:122)
	at __randomizedtesting.SeedInfo.seed([BF0290866116A66E]:0)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Error from server at https://127.0.0.1:34177/bk_igy: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:34177/bk_igy: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:560)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:325)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:442)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:188)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9770 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/init-core-data-001
   [junit4]   2> 67391 T206 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 67392 T206 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 68311 T206 oahu.NativeCodeLoader.<clinit> WARN Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 69354 T206 oahmi.MetricsConfig.loadFirst WARN Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 69700 T206 oml.Slf4jLog.info Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
   [junit4]   2> 69711 T206 oahh.HttpRequestLog.getRequestLog WARN Jetty request log can only be enabled using Log4j
   [junit4]   2> 69802 T206 oml.Slf4jLog.info jetty-6.1.26
   [junit4]   2> 69856 T206 oml.Slf4jLog.info Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.6.0-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_51673_hdfs____z83g9s/webapp
   [junit4]   2> 70019 T206 oml.Slf4jLog.info NO JSP Support for /, did not find org.apache.jasper.servlet.JspServlet
   [junit4]   2> 71065 T206 oml.Slf4jLog.info Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:51673
   [junit4]   2> 72201 T206 oahh.HttpRequestLog.getRequestLog WARN Jetty request log can only be enabled using Log4j
   [junit4]   2> 72208 T206 oml.Slf4jLog.info jetty-6.1.26
   [junit4]   2> 72220 T206 oml.Slf4jLog.info Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.6.0-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_48555_datanode____46qj94/webapp
   [junit4]   2> 72331 T206 oml.Slf4jLog.info NO JSP Support for /, did not find org.apache.jasper.servlet.JspServlet
   [junit4]   2> 72748 T206 oml.Slf4jLog.info Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48555
   [junit4]   2> 73034 T206 oahh.HttpRequestLog.getRequestLog WARN Jetty request log can only be enabled using Log4j
   [junit4]   2> 73036 T206 oml.Slf4jLog.info jetty-6.1.26
   [junit4]   2> 73051 T206 oml.Slf4jLog.info Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.6.0-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_36454_datanode____.ldxz0p/webapp
   [junit4]   2> 73156 T206 oml.Slf4jLog.info NO JSP Support for /, did not find org.apache.jasper.servlet.JspServlet
   [junit4]   2> 73617 T206 oml.Slf4jLog.info Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36454
   [junit4]   2> 74644 T239 oahhsb.BlockManager.processReport BLOCK* processReport: from storage DS-1a28c5aa-cb47-4c53-937d-f97e4e4a62c2 node DatanodeRegistration(127.0.0.1, datanodeUuid=3771a3c3-74fe-446c-9764-a9d426772d57, infoPort=36454, ipcPort=53888, storageInfo=lv=-56;cid=testClusterID;nsid=980201884;c=0), blocks: 0, hasStaleStorages: true, processing time: 3 msecs
   [junit4]   2> 74644 T240 oahhsb.BlockManager.processReport BLOCK* processReport: from storage DS-2118ff69-db6c-40a0-995d-22bf25763cce node DatanodeRegistration(127.0.0.1, datanodeUuid=2c83e650-504c-40f3-b0fe-2bb5ed6e5e8f, infoPort=48555, ipcPort=57044, storageInfo=lv=-56;cid=testClusterID;nsid=980201884;c=0), blocks: 0, hasStaleStorages: true, processing time: 1 msecs
   [junit4]   2> 74652 T239 oahhsb.BlockManager.processReport BLOCK* processReport: from storage DS-a6b84626-99a8-4208-b659-f08406fe8681 node DatanodeRegistration(127.0.0.1, datanodeUuid=3771a3c3-74fe-446c-9764-a9d426772d57, infoPort=36454, ipcPort=53888, storageInfo=lv=-56;cid=testClusterID;nsid=980201884;c=0), blocks: 0, hasStaleStorages: false, processing time: 0 msecs
   [junit4]   2> 74653 T240 oahhsb.BlockManager.processReport BLOCK* processReport: from storage DS-47821c0c-5c2d-4b35-a689-26cc0a90210d node DatanodeRegistration(127.0.0.1, datanodeUuid=2c83e650-504c-40f3-b0fe-2bb5ed6e5e8f, infoPort=48555, ipcPort=57044, storageInfo=lv=-56;cid=testClusterID;nsid=980201884;c=0), blocks: 0, hasStaleStorages: false, processing time: 0 msecs
   [junit4]   2> 75171 T206 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 75172 T334 oasc.ZkTestServer$2$1.setClientPort client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 75172 T334 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 75272 T206 oasc.ZkTestServer.run start zk server on port:46807
   [junit4]   2> 75273 T206 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 75275 T206 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75279 T341 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58a11c90 name:ZooKeeperConnection Watcher:127.0.0.1:46807 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75279 T206 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75280 T206 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 75280 T206 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 75291 T206 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 75303 T206 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 75310 T344 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@344d312e name:ZooKeeperConnection Watcher:127.0.0.1:46807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 75310 T206 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 75311 T206 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 75314 T206 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 75322 T206 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 75326 T206 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 75329 T206 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 75331 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 75332 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 75335 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 75336 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 75339 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 75340 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 75342 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 75342 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 75345 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 75345 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 75348 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 75348 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 75350 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 75351 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 75353 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 75354 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 75356 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 75356 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 75358 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 75359 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 75361 T206 oasc.AbstractZkTestCase.putConfig put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 75361 T206 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 76109 T206 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores/collection1
   [junit4]   2> 76120 T206 oejs.Server.doStart jetty-9.2.10.v20150310
   [junit4]   2> 76133 T206 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@fdec100{/,null,AVAILABLE}
   [junit4]   2> 76134 T206 oejs.AbstractConnector.doStart Started ServerConnector@1078427b{HTTP/1.1}{127.0.0.1:57018}
   [junit4]   2> 76135 T206 oejs.Server.doStart Started @77819ms
   [junit4]   2> 76135 T206 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=hdfs://localhost:36935/hdfs__localhost_36935__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-trunk_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_BF0290866116A66E-001_tempDir-002_control_data, hostContext=/, hostPort=57018, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores}
   [junit4]   2> 76136 T206 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
   [junit4]   2> 76144 T206 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/'
   [junit4]   2> 76163 T206 oasc.SolrXmlConfig.fromFile Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/solr.xml
   [junit4]   2> 76174 T206 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores
   [junit4]   2> 76174 T206 oasc.CoreContainer.<init> New CoreContainer 274754022
   [junit4]   2> 76175 T206 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/]
   [junit4]   2> 76175 T206 oasc.CoreContainer.load loading shared library: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/lib
   [junit4]   2> 76175 T206 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/lib).
   [junit4]   2> 76183 T206 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 76185 T206 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 76186 T206 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 76186 T206 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 76186 T206 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 76186 T206 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:46807/solr
   [junit4]   2> 76203 T206 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 76203 T206 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 76204 T206 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 76232 T358 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29d81121 name:ZooKeeperConnection Watcher:127.0.0.1:46807 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 76240 T206 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 76240 T206 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 76276 T206 n:127.0.0.1:57018_ oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 76304 T361 n:127.0.0.1:57018_ oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69a5a415 name:ZooKeeperConnection Watcher:127.0.0.1:46807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 76304 T206 n:127.0.0.1:57018_ oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 76323 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 76331 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 76342 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 76346 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 76349 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 76377 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 76380 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 76382 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 76383 T206 n:127.0.0.1:57018_ oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57018_
   [junit4]   2> 76384 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57018_
   [junit4]   2> 76388 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 76391 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 76395 T206 n:127.0.0.1:57018_ oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 76409 T206 n:127.0.0.1:57018_ oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:57018_
   [junit4]   2> 76410 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 76412 T206 n:127.0.0.1:57018_ oasc.Overseer.start Overseer (id=93894552584585219-127.0.0.1:57018_-n_0000000000) starting
   [junit4]   2> 76430 T206 n:127.0.0.1:57018_ oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 76486 T206 n:127.0.0.1:57018_ oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 76508 T363 n:127.0.0.1:57018_ oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 76508 T206 n:127.0.0.1:57018_ oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 76497 T362 n:127.0.0.1:57018_ oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 76537 T206 n:127.0.0.1:57018_ oasc.CoreContainer.initializeAuthenticationPlugin No authentication plugin used.
   [junit4]   2> 76538 T206 n:127.0.0.1:57018_ oasc.CoreContainer.intializeAuthorizationPlugin Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 76539 T206 n:127.0.0.1:57018_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores
   [junit4]   2> 76543 T206 n:127.0.0.1:57018_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 76544 T206 n:127.0.0.1:57018_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores/collection1/
   [junit4]   2> 76545 T206 n:127.0.0.1:57018_ oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 76568 T365 n:127.0.0.1:57018_ c:control_collection x:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 76569 T365 n:127.0.0.1:57018_ c:control_collection x:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 76584 T361 n:127.0.0.1:57018_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 76584 T365 n:127.0.0.1:57018_ oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 76587 T362 n:127.0.0.1:57018_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57018",
   [junit4]   2> 	  "node_name":"127.0.0.1:57018_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 76613 T362 n:127.0.0.1:57018_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57018",
   [junit4]   2> 	  "node_name":"127.0.0.1:57018_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 76615 T362 n:127.0.0.1:57018_ oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 76627 T362 n:127.0.0.1:57018_ oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 76630 T361 n:127.0.0.1:57018_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 77585 T365 n:127.0.0.1:57018_ oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 77585 T365 n:127.0.0.1:57018_ oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 77586 T365 n:127.0.0.1:57018_ oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 77587 T365 n:127.0.0.1:57018_ oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 77588 T365 n:127.0.0.1:57018_ oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 77588 T365 n:127.0.0.1:57018_ oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores/collection1/'
   [junit4]   2> 77604 T365 n:127.0.0.1:57018_ oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 77613 T365 n:127.0.0.1:57018_ oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 77617 T365 n:127.0.0.1:57018_ oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 77624 T365 n:127.0.0.1:57018_ oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 77626 T365 n:127.0.0.1:57018_ oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 77635 T365 n:127.0.0.1:57018_ oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 77745 T365 n:127.0.0.1:57018_ oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 77746 T365 n:127.0.0.1:57018_ oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 77747 T365 n:127.0.0.1:57018_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 77753 T365 n:127.0.0.1:57018_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 77803 T365 n:127.0.0.1:57018_ oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 77804 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.HdfsDirectoryFactory
   [junit4]   2> 77823 T365 n:127.0.0.1:57018_ x:collection1 oasc.HdfsDirectoryFactory.init solr.hdfs.home=hdfs://localhost:36935/solr_hdfs_home
   [junit4]   2> 77823 T365 n:127.0.0.1:57018_ x:collection1 oasc.HdfsDirectoryFactory.init Solr Kerberos Authentication disabled
   [junit4]   2> 77829 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrCore.<init> [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/control-001/cores/collection1/], dataDir=[null]
   [junit4]   2> 77832 T365 n:127.0.0.1:57018_ x:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a59a4e
   [junit4]   2> 77840 T365 n:127.0.0.1:57018_ x:collection1 oasc.HdfsDirectoryFactory.create creating directory factory for path hdfs://localhost:36935/solr_hdfs_home/control_collection/core_node1/data
   [junit4]   2> 77956 T365 n:127.0.0.1:57018_ x:collection1 oasc.CachingDirectoryFactory.get return new directory for hdfs://localhost:36935/solr_hdfs_home/control_collection/core_node1/data
   [junit4]   2> 77962 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=hdfs://localhost:36935/solr_hdfs_home/control_collection/core_node1/data/index/
   [junit4]   2> 78002 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory 'hdfs:/localhost:36935/solr_hdfs_home/control_collection/core_node1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 78002 T365 n:127.0.0.1:57018_ x:collection1 oasc.HdfsDirectoryFactory.create creating directory factory for path hdfs://localhost:36935/solr_hdfs_home/control_collection/core_node1/data/index
   [junit4]   2> 78012 T365 n:127.0.0.1:57018_ x:collection1 oasc.HdfsDirectoryFactory.create Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 78012 T365 n:127.0.0.1:57018_ x:collection1 oasc.HdfsDirectoryFactory.create Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 78012 T365 n:127.0.0.1:57018_ x:collection1 oasc.HdfsDirectoryFactory.getBlockDirectoryCache Creating new single instance HDFS BlockCache
   [junit4]   2> 79931 T365 n:127.0.0.1:57018_ x:collection1 oassb.BlockDirectory.<init> Block cache on write is disabled
   [junit4]   2> 79932 T365 n:127.0.0.1:57018_ x:collection1 oasc.CachingDirectoryFactory.get return new directory for hdfs://localhost:36935/solr_hdfs_home/control_collection/core_node1/data/index
   [junit4]   2> 79932 T365 n:127.0.0.1:57018_ x:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=89.0146484375, floorSegmentMB=0.638671875, forceMergeDeletesPctAllowed=12.969966594368387, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6480391641503377
   [junit4]   2> 80317 T238 oahhsb.BlockManager.logAddStoredBlock BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41523 is added to blk_1073741825_1001{blockUCState=COMMITTED, primaryNodeIndex=-1, replicas=[ReplicaUnderConstruction[[DISK]DS-1a28c5aa-cb47-4c53-937d-f97e4e4a62c2:NORMAL:127.0.0.1:48539|RBW], ReplicaUnderConstruction[[DISK]DS-47821c0c-5c2d-4b35-a689-26cc0a90210d:NORMAL:127.0.0.1:41523|RBW]]} size 68
   [junit4]   2> 80318 T237 oahhsb.BlockManager.logAddStoredBlock BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48539 is added to blk_1073741825_1001 size 68
   [junit4]   2> 80721 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=NRTCachingDirectory(BlockDirectory(HdfsDirectory@76f2b187 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3ed6c94f); maxCacheMB=192.0 maxMergeSizeMB=16.0),segFN=segments_1,generation=1}
   [junit4]   2> 80722 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 80727 T365 n:127.0.0.1:57018_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 80737 T365 n:127.0.0.1:57018_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 80738 T365 n:127.0.0.1:57018_ x:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 80738 T365 n:127.0.0.1:57018_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 80738 T365 n:127.0.0.1:57018_ x:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 80738 T365 n:127.0.0.1:57018_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 80739 T365 n:127.0.0.1:57018_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 80739 T365 n:127.0.0.1:57018_ x:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 80739 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 80741 T365 n:127.0.0.1:57018_ x:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 80742 T365 n:127.0.0.1:57018_ x:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 80743 T365 n:127.0.0.1:57018_ x:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 80744 T365 n:127.0.0.1:57018_ x:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 80748 T365 n:127.0.0.1:57018_ x:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
   [junit4]   2> 80748 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 80762 T365 n:127.0.0.1:57018_ x:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 80762 T365 n:127.0.0.1:57018_ x:collection1 oasu.HdfsUpdateLog.init Initializing HdfsUpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 tlogDfsReplication=2
   [junit4]   2> 80790 T365 n:127.0.0.1:57018_ x:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 80790 T365 n:127.0.0.1:57018_ x:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 80796 T365 n:127.0.0.1:57018_ x:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=2.3115234375, floorSegmentMB=0.5283203125, forceMergeDeletesPctAllowed=26.33144623317907, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3763675730763254
   [junit4]   2> 80893 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=NRTCachingDirectory(BlockDirectory(HdfsDirectory@76f2b187 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3ed6c94f); maxCacheMB=192.0 maxMergeSizeMB=16.0),segFN=segments_1,generation=1}
   [junit4]   2> 80894 T365 n:127.0.0.1:57018_ x:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 80894 T365 n:127.0.0.1:57018_ x:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@41143a5c[collection1] main
   [junit4]   2> 80894 T365 n:127.0.0.1:57018_ x:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 80896 T365 n:127.0.0.1:57018_ x:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 80896 T365 n:127.0.0.1:57018_ x:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 80899 T365 n:127.0.0.1:57018_ x:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 80899 T365 n:127.0.0.1:57018_ x:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 80899 T365 n:127.0.0.1:57018_ x:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 80900 T365 n:127.0.0.1:57018_ x:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 80901 T365 n:127.0.0.1:57018_ x:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 80901 T365 n:127.0.0.1:57018_ x:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 80901 T365 n:127.0.0.1:57018_ x:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 80903 T372 n:127.0.0.1:57018_ x:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@41143a5c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 80905 T365 n:127.0.0.1:57018_ x:collection1 oasc.ZkController.getConfDirListeners watch zkdir /configs/conf1
   [junit4]   2> 80907 T365 n:127.0.0.1:57018_ x:collection1 oasu.UpdateLog.onFirstSearcher On first searcher opened, looking up max value of version field
   [junit4]   2> 80907 T365 n:127.0.0.1:57018_ x:collection1 oasu.VersionInfo.getMaxVersionFromIndex Refreshing highest value of _version_ for 256 version buckets from index
   [junit4]   2> 80907 T365 n:127.0.0.1:57018_ x:collection1 oasu.VersionInfo.getMaxVersionFromIndex No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 80907 T365 n:127.0.0.1:57018_ x:collection1 oasu.UpdateLog.seedBucketsWithHighestVersion Could not find max version in index or recent updates, using new clock 1502312847362752512
   [junit4]   2> 80907 T365 n:127.0.0.1:57018_ x:collection1 oasu.UpdateLog.seedBucketsWithHighestVersion Took 0 ms to seed version buckets with highest version 1502312847362752512
   [junit4]   2> 80908 T365 n:127.0.0.1:57018_ x:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 80909 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57018 collection:control_collection shard:shard1
   [junit4]   2> 80909 T206 n:127.0.0.1:57018_ oass.SolrDispatchFilter.init user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2
   [junit4]   2> 80909 T206 n:127.0.0.1:57018_ oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 80930 T206 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 80931 T206 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 80931 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 80934 T380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ee33c20 name:ZooKeeperConnection Watcher:127.0.0.1:46807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 80935 T206 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 80935 T206 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 80935 T206 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 80943 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 80945 T361 n:127.0.0.1:57018_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 80948 T362 n:127.0.0.1:57018_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 80948 T206 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 80948 T206 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 80949 T206 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 80952 T206 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 80952 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 80957 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C35 name=collection1 org.apache.solr.core.SolrCore@6167a016 url=http://127.0.0.1:57018/collection1 node=127.0.0.1:57018_ C35_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:57018, node_name=127.0.0.1:57018_, state=down}
   [junit4]   2> 80957 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 C35 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57018/collection1/
   [junit4]   2> 80958 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 C35 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 80958 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 C35 oasc.SyncStrategy.syncToMe http://127.0.0.1:57018/collection1/ has no replicas
   [junit4]   2> 80959 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57018/collection1/ shard1
   [junit4]   2> 80956 T380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 80958 T361 n:127.0.0.1:57018_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 80960 T383 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5aa0f4f8 name:ZooKeeperConnection Watcher:127.0.0.1:46807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 80960 T206 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 80961 T206 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 80970 T335 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14d94991d260005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 80971 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 80984 T384 n:127.0.0.1:57018_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 80987 T362 n:127.0.0.1:57018_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"5",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 80987 T362 n:127.0.0.1:57018_ oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 80995 T362 n:127.0.0.1:57018_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57018",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 80996 T362 n:127.0.0.1:57018_ oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 81004 T380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 81005 T384 n:127.0.0.1:57018_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 81012 T384 n:127.0.0.1:57018_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 81020 T380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 81020 T384 n:127.0.0.1:57018_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 81045 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ZkController.register We are http://127.0.0.1:57018/collection1/ and leader is http://127.0.0.1:57018/collection1/
   [junit4]   2> 81045 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57018
   [junit4]   2> 81045 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 81045 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 81045 T377 n:127.0.0.1:57018_ c:control_collection s:shard1 x:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 81047 T384 n:127.0.0.1:57018_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 81048 T362 n:127.0.0.1:57018_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57018",
   [junit4]   2> 	  "node_name":"127.0.0.1:57018_",
   [junit4]   2> 	  "numShards":"5",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 81049 T362 n:127.0.0.1:57018_ oasco.ReplicaMutator.updateState Update state numShards=5 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57018",
   [junit4]   2> 	  "node_name":"127.0.0.1:57018_",
   [junit4]   2> 	  "numShards":"5",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 81155 T380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 81159 T384 n:127.0.0.1:57018_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 81385 T206 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores/collection1
   [junit4]   2> 81386 T206 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001
   [junit4]   2> 81387 T206 oejs.Server.doStart jetty-9.2.10.v20150310
   [junit4]   2> 81390 T206 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@2daf2620{/,null,AVAILABLE}
   [junit4]   2> 81391 T206 oejs.AbstractConnector.doStart Started ServerConnector@e693fb7{HTTP/1.1}{127.0.0.1:56531}
   [junit4]   2> 81391 T206 oejs.Server.doStart Started @83075ms
   [junit4]   2> 81391 T206 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=hdfs://localhost:36935/hdfs__localhost_36935__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-trunk_solr_build_solr-core_test_J2_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_BF0290866116A66E-001_tempDir-002_jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=56531, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores}
   [junit4]   2> 81392 T206 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
   [junit4]   2> 81392 T206 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/'
   [junit4]   2> 81405 T206 oasc.SolrXmlConfig.fromFile Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/solr.xml
   [junit4]   2> 81411 T206 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores
   [junit4]   2> 81411 T206 oasc.CoreContainer.<init> New CoreContainer 1835379279
   [junit4]   2> 81412 T206 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/]
   [junit4]   2> 81412 T206 oasc.CoreContainer.load loading shared library: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/lib
   [junit4]   2> 81412 T206 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/lib).
   [junit4]   2> 81419 T206 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 81420 T206 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 81421 T206 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 81421 T206 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 81422 T206 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 81422 T206 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:46807/solr
   [junit4]   2> 81422 T206 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 81422 T206 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 81423 T206 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 81434 T398 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26315792 name:ZooKeeperConnection Watcher:127.0.0.1:46807 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 81434 T206 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 81435 T206 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 81437 T206 n:127.0.0.1:56531_ oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 81441 T401 n:127.0.0.1:56531_ oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a3e41c2 name:ZooKeeperConnection Watcher:127.0.0.1:46807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 81441 T206 n:127.0.0.1:56531_ oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 81447 T206 n:127.0.0.1:56531_ oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 82458 T206 n:127.0.0.1:56531_ oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56531_
   [junit4]   2> 82459 T206 n:127.0.0.1:56531_ oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56531_
   [junit4]   2> 82464 T206 n:127.0.0.1:56531_ oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 82467 T206 n:127.0.0.1:56531_ oasc.CoreContainer.initializeAuthenticationPlugin No authentication plugin used.
   [junit4]   2> 82476 T206 n:127.0.0.1:56531_ oasc.CoreContainer.intializeAuthorizationPlugin Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 82477 T206 n:127.0.0.1:56531_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores
   [junit4]   2> 82478 T206 n:127.0.0.1:56531_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores/collection1, collection=collection1, absoluteInstDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 82478 T206 n:127.0.0.1:56531_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores/collection1/
   [junit4]   2> 82478 T206 n:127.0.0.1:56531_ oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 82489 T402 n:127.0.0.1:56531_ c:collection1 x:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 82490 T402 n:127.0.0.1:56531_ c:collection1 x:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 82492 T384 n:127.0.0.1:57018_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 82493 T362 n:127.0.0.1:57018_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56531",
   [junit4]   2> 	  "node_name":"127.0.0.1:56531_",
   [junit4]   2> 	  "numShards":"5",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 82494 T362 n:127.0.0.1:57018_ oasco.ReplicaMutator.updateState Update state numShards=5 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56531",
   [junit4]   2> 	  "node_name":"127.0.0.1:56531_",
   [junit4]   2> 	  "numShards":"5",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 82494 T362 n:127.0.0.1:57018_ oasco.ReplicaMutator.updateState Collection already exists with numShards=5
   [junit4]   2> 82495 T362 n:127.0.0.1:57018_ oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 82496 T402 n:127.0.0.1:56531_ oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 82496 T402 n:127.0.0.1:56531_ oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 82498 T402 n:127.0.0.1:56531_ oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 82508 T402 n:127.0.0.1:56531_ oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 82509 T402 n:127.0.0.1:56531_ oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 82599 T362 n:127.0.0.1:57018_ oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 82600 T401 n:127.0.0.1:56531_ oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 82601 T401 n:127.0.0.1:56531_ oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 83509 T402 n:127.0.0.1:56531_ oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 83509 T402 n:127.0.0.1:56531_ oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 83510 T402 n:127.0.0.1:56531_ oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 83510 T402 n:127.0.0.1:56531_ oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 83511 T402 n:127.0.0.1:56531_ oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 83512 T402 n:127.0.0.1:56531_ oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores/collection1/'
   [junit4]   2> 83530 T402 n:127.0.0.1:56531_ oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 83544 T402 n:127.0.0.1:56531_ oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 83548 T402 n:127.0.0.1:56531_ oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 83556 T402 n:127.0.0.1:56531_ oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 83557 T402 n:127.0.0.1:56531_ oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 83562 T402 n:127.0.0.1:56531_ oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 83665 T402 n:127.0.0.1:56531_ oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 83667 T402 n:127.0.0.1:56531_ oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 83678 T402 n:127.0.0.1:56531_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 83681 T402 n:127.0.0.1:56531_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 83762 T402 n:127.0.0.1:56531_ oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 83763 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.HdfsDirectoryFactory
   [junit4]   2> 83763 T402 n:127.0.0.1:56531_ x:collection1 oasc.HdfsDirectoryFactory.init solr.hdfs.home=hdfs://localhost:36935/solr_hdfs_home
   [junit4]   2> 83766 T402 n:127.0.0.1:56531_ x:collection1 oasc.HdfsDirectoryFactory.init Solr Kerberos Authentication disabled
   [junit4]   2> 83766 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrCore.<init> [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest BF0290866116A66E-001/shard-1-001/cores/collection1/], dataDir=[null]
   [junit4]   2> 83766 T402 n:127.0.0.1:56531_ x:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a59a4e
   [junit4]   2> 83767 T402 n:127.0.0.1:56531_ x:collection1 oasc.HdfsDirectoryFactory.create creating directory factory for path hdfs://localhost:36935/solr_hdfs_home/collection1/core_node1/data
   [junit4]   2> 83826 T402 n:127.0.0.1:56531_ x:collection1 oasc.CachingDirectoryFactory.get return new directory for hdfs://localhost:36935/solr_hdfs_home/collection1/core_node1/data
   [junit4]   2> 83827 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=hdfs://localhost:36935/solr_hdfs_home/collection1/core_node1/data/index/
   [junit4]   2> 83861 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory 'hdfs:/localhost:36935/solr_hdfs_home/collection1/core_node1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 83862 T402 n:127.0.0.1:56531_ x:collection1 oasc.HdfsDirectoryFactory.create creating directory factory for path hdfs://localhost:36935/solr_hdfs_home/collection1/core_node1/data/index
   [junit4]   2> 83877 T402 n:127.0.0.1:56531_ x:collection1 oasc.HdfsDirectoryFactory.create Number of slabs of block cache [1] with direct memory allocation set to [true]
   [junit4]   2> 83878 T402 n:127.0.0.1:56531_ x:collection1 oasc.HdfsDirectoryFactory.create Block cache target memory usage, slab size of [8388608] will allocate [1] slabs and use ~[8388608] bytes
   [junit4]   2> 83878 T402 n:127.0.0.1:56531_ x:collection1 oasc.HdfsDirectoryFactory.getBlockDirectoryCache Creating new single instance HDFS BlockCache
   [junit4]   2> 83911 T402 n:127.0.0.1:56531_ x:collection1 oassb.BlockDirectory.<init> Block cache on write is disabled
   [junit4]   2> 83916 T402 n:127.0.0.1:56531_ x:collection1 oasc.CachingDirectoryFactory.get return new directory for hdfs://localhost:36935/solr_hdfs_home/collection1/core_node1/data/index
   [junit4]   2> 83916 T402 n:127.0.0.1:56531_ x:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=89.0146484375, floorSegmentMB=0.638671875, forceMergeDeletesPctAllowed=12.969966594368387, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6480391641503377
   [junit4]   2> 83985 T239 oahhsb.BlockManager.logAddStoredBlock BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:41523 is added to blk_1073741826_1002{blockUCState=UNDER_CONSTRUCTION, primaryNodeIndex=-1, replicas=[ReplicaUnderConstruction[[DISK]DS-a6b84626-99a8-4208-b659-f08406fe8681:NORMAL:127.0.0.1:48539|RBW], ReplicaUnderConstruction[[DISK]DS-47821c0c-5c2d-4b35-a689-26cc0a90210d:NORMAL:127.0.0.1:41523|RBW]]} size 0
   [junit4]   2> 83996 T237 oahhsb.BlockManager.logAddStoredBlock BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:48539 is added to blk_1073741826_1002{blockUCState=UNDER_CONSTRUCTION, primaryNodeIndex=-1, replicas=[ReplicaUnderConstruction[[DISK]DS-47821c0c-5c2d-4b35-a689-26cc0a90210d:NORMAL:127.0.0.1:41523|RBW], ReplicaUnderConstruction[[DISK]DS-1a28c5aa-cb47-4c53-937d-f97e4e4a62c2:NORMAL:127.0.0.1:48539|FINALIZED]]} size 0
   [junit4]   2> 84004 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=NRTCachingDirectory(BlockDirectory(HdfsDirectory@45eceeab lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@336fab0a); maxCacheMB=192.0 maxMergeSizeMB=16.0),segFN=segments_1,generation=1}
   [junit4]   2> 84004 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 84010 T402 n:127.0.0.1:56531_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 84010 T402 n:127.0.0.1:56531_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 84011 T402 n:127.0.0.1:56531_ x:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 84011 T402 n:127.0.0.1:56531_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 84011 T402 n:127.0.0.1:56531_ x:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 84011 T402 n:127.0.0.1:56531_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 84012 T402 n:127.0.0.1:56531_ x:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 84012 T402 n:127.0.0.1:56531_ x:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 84012 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 84014 T402 n:127.0.0.1:56531_ x:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 84015 T402 n:127.0.0.1:56531_ x:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 84016 T402 n:127.0.0.1:56531_ x:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 84017 T402 n:127.0.0.1:56531_ x:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 84030 T402 n:127.0.0.1:56531_ x:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
   [junit4]   2> 84031 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 84036 T402 n:127.0.0.1:56531_ x:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 84036 T402 n:127.0.0.1:56531_ x:collection1 oasu.HdfsUpdateLog.init Initializing HdfsUpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 tlogDfsReplication=2
   [junit4]   2> 84062 T402 n:127.0.0.1:56531_ x:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 84062 T402 n:127.0.0.1:56531_ x:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 84065 T402 n:127.0.0.1:56531_ x:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=2.3115234375, floorSegmentMB=0.5283203125, forceMergeDeletesPctAllowed=26.33144623317907, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3763675730763254
   [junit4]   2> 84097 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=NRTCachingDirectory(BlockDirectory(HdfsDirectory@45eceeab lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@336fab0a); maxCacheMB=192.0 maxMergeSizeMB=16.0),segFN=segments_1,generation=1}
   [junit4]   2> 84097 T402 n:127.0.0.1:56531_ x:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 84098 T402 n:127.0.0.1:56531_ x:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@664da18c[collection1] main
   [junit4]   2> 84098 T402 n:127.0.0.1:56531_ x:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 84099 T402 n:127.0.0.1:56531_ x:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 84100 T402 n:127.0.0.1:56531_ x:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 84100 T402 n:127.0.0.1:56531_ x:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 84100 T402 n:127.0.0.1:56531_ x:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 84100 T402 n:127.0.0.1:56531_ x:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 84101 T402 n:127.0.0.1:56531_ x:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 84101 T402 n:127.0.0.1:56531_ x:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 84101 T402 n:127.0.0.1:56531_ x:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 84102 T402 n:127.0.0.1:56531_ x:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4

[...truncated too long message...]

s-trunk/solr/server/lib/jetty-server-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-servlet-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-servlets-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-util-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-webapp-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-xml-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/example/example-DIH/solr/db/lib/derby-10.9.1.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/build/core/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/test-framework/lib/junit-4.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/test-framework/lib/randomizedtesting-runner-2.1.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/apacheds-all-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/commons-collections-3.2.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/hadoop-common-2.6.0-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/hadoop-hdfs-2.6.0-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/hadoop-minikdc-2.6.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jcl-over-slf4j-1.7.7.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/build/analysis/icu/lucene-analyzers-icu-6.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/contrib/analysis-extras/lib/icu4j-54.1.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-antlr.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-regexp.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-resolver.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-commons-net.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jmf.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-oro.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-xalan2.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-commons-logging.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jsch.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-junit4.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-javamail.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jai.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jdepend.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-junit.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-bsf.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-swing.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-netrexx.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-bcel.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-testutil.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-log4j.jar:/x1/jenkins/jenkins-slave/tools/hudson.model.JDK/latest1.8/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.1.13.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/temp/junit4-J1-20150527_045815_088.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/temp/junit4-J1-20150527_045815_088.suites
   [junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1504)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:133)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:964)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:961)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:533: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:474: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:61: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/extra-targets.xml:39: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build.xml:230: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/common-build.xml:512: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:1415: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:973: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 407 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No artifacts from Lucene-Solr-NightlyTests-trunk #679 to compare, so performing full copy of artifacts
ERROR: Failed to archive artifacts: **/*.events,heapdumps/**,**/hs_err_pid*,README.txt
java.io.IOException: Failed to extract /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/transfer of 4 files
	at hudson.FilePath.readFromTar(FilePath.java:2101)
	at hudson.FilePath.copyRecursiveTo(FilePath.java:2013)
	at jenkins.model.StandardArtifactManager.archive(StandardArtifactManager.java:61)
	at com.cloudbees.jenkins.plugins.jsync.archiver.JSyncArtifactManager.remoteSync(JSyncArtifactManager.java:114)
	at com.cloudbees.jenkins.plugins.jsync.archiver.JSyncArtifactManager.archive(JSyncArtifactManager.java:68)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:140)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:756)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:720)
	at hudson.model.Build$BuildExecution.post2(Build.java:182)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:669)
	at hudson.model.Run.execute(Run.java:1731)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:88)
	at hudson.model.Executor.run(Executor.java:232)
Caused by: java.io.IOException: unexpected EOF with 1024 bytes unread
	at hudson.org.apache.tools.tar.TarInputStream.read(TarInputStream.java:349)
	at java.io.FilterInputStream.read(FilterInputStream.java:107)
	at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:1792)
	at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:1769)
	at org.apache.commons.io.IOUtils.copy(IOUtils.java:1744)
	at hudson.util.IOUtils.copy(IOUtils.java:40)
	at hudson.FilePath.readFromTar(FilePath.java:2091)
	... 14 more
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-trunk - Build # 696 - Still Failing

Posted by Chris Hostetter <ho...@fucit.org>.
I've updated SOLR-7603 with the new details from the 
UpdateRequestProcessorFactoryTest failure and will try to make sense of 
that.

Anybody have any clue what's up with the HttpPartitionTest test here?

leaking file handles on the segments file?


: Date: Fri, 29 May 2015 09:42:29 +0000 (UTC)
: From: Apache Jenkins Server <je...@builds.apache.org>
: Reply-To: dev@lucene.apache.org
: To: dev@lucene.apache.org
: Subject: [JENKINS] Lucene-Solr-NightlyTests-trunk - Build # 696 - Still
:     Failing
: 
: Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-trunk/696/
: 
: 3 tests failed.
: FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.HttpPartitionTest
: 
: Error Message:
: file handle leaks: [SeekableByteChannel(/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 3AF6852C6379681-001/index-SimpleFSDirectory-011/segments_2)]
: 
: Stack Trace:
: java.lang.RuntimeException: file handle leaks: [SeekableByteChannel(/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 3AF6852C6379681-001/index-SimpleFSDirectory-011/segments_2)]
: 	at __randomizedtesting.SeedInfo.seed([3AF6852C6379681]:0)
: 	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
: 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
: 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
: 	at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:227)
: 	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
: 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
: 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
: 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
: 	at java.lang.Thread.run(Thread.java:745)
: Caused by: java.lang.Exception
: 	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
: 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:82)
: 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:272)
: 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:213)
: 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:241)
: 	at java.nio.file.Files.newByteChannel(Files.java:361)
: 	at java.nio.file.Files.newByteChannel(Files.java:407)
: 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:76)
: 	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:110)
: 	at org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:42)
: 	at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:269)
: 	at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:488)
: 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:278)
: 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:265)
: 	at org.apache.lucene.store.BaseDirectoryWrapper.close(BaseDirectoryWrapper.java:46)
: 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:307)
: 	at org.apache.solr.core.CachingDirectoryFactory.closeCacheValue(CachingDirectoryFactory.java:273)
: 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:203)
: 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1254)
: 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:311)
: 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:198)
: 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)
: 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)
: 	at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)
: 	at org.apache.solr.cloud.ZkController.register(ZkController.java:884)
: 	at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
: 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
: 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
: 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
: 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
: 	... 1 more
: 
: 
: REGRESSION:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping
: 
: Error Message:
: distrib-chain-explicit expected LogUpdateProcessor in chain due to @RunAllways, but not found: org.apache.solr.update.processor.DistributedUpdateProcessor@638c6e19, org.apache.solr.update.processor.RemoveBlankFieldUpdateProcessorFactory$1@25bbdfcb, org.apache.solr.update.processor.RunUpdateProcessor@473fea05, 
: 
: Stack Trace:
: java.lang.AssertionError: distrib-chain-explicit expected LogUpdateProcessor in chain due to @RunAllways, but not found: org.apache.solr.update.processor.DistributedUpdateProcessor@638c6e19, org.apache.solr.update.processor.RemoveBlankFieldUpdateProcessorFactory$1@25bbdfcb, org.apache.solr.update.processor.RunUpdateProcessor@473fea05, 
: 	at __randomizedtesting.SeedInfo.seed([3AF6852C6379681:724B9684B0DCB14D]:0)
: 	at org.junit.Assert.fail(Assert.java:93)
: 	at org.junit.Assert.assertTrue(Assert.java:43)
: 	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:127)
: 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
: 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
: 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
: 	at java.lang.reflect.Method.invoke(Method.java:483)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
: 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
: 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
: 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
: 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
: 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
: 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
: 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
: 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
: 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
: 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
: 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
: 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
: 	at java.lang.Thread.run(Thread.java:745)
: 
: 
: FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test
: 
: Error Message:
: Captured an uncaught exception in thread: Thread[id=3840, name=collection5, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
: 
: Stack Trace:
: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3840, name=collection5, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
: Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44466/xk/t: collection already exists: awholynewstresscollection_collection5_1
: 	at __randomizedtesting.SeedInfo.seed([3AF6852C6379681]:0)
: 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:560)
: 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
: 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
: 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:376)
: 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:328)
: 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
: 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:856)
: 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:799)
: 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
: 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1570)
: 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1591)
: 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:860)
: 
: 
: 
: 
: Build Log:
: [...truncated 10189 lines...]
:    [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
:    [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/init-core-data-001
:    [junit4]   2> 401841 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3AF6852C6379681]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true)
:    [junit4]   2> 401841 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3AF6852C6379681]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /xk/t
:    [junit4]   2> 401844 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
:    [junit4]   2> 401845 INFO  (Thread-2385) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
:    [junit4]   2> 401845 INFO  (Thread-2385) [    ] o.a.s.c.ZkTestServer Starting server
:    [junit4]   2> 401945 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkTestServer start zk server on port:38050
:    [junit4]   2> 401959 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
:    [junit4]   2> 401961 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
:    [junit4]   2> 401963 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
:    [junit4]   2> 401965 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
:    [junit4]   2> 401967 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
:    [junit4]   2> 401968 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
:    [junit4]   2> 401970 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
:    [junit4]   2> 401971 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
:    [junit4]   2> 401973 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
:    [junit4]   2> 401974 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
:    [junit4]   2> 401976 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
:    [junit4]   2> 402265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1
:    [junit4]   2> 402267 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.Server jetty-9.2.10.v20150310
:    [junit4]   2> 402269 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2bd76d64{/xk/t,null,AVAILABLE}
:    [junit4]   2> 402288 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.ServerConnector Started ServerConnector@19fab22b{HTTP/1.1}{127.0.0.1:60774}
:    [junit4]   2> 402288 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.Server Started @405784ms
:    [junit4]   2> 402289 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/tempDir-001/control/data, hostContext=/xk/t, hostPort=60774, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores}
:    [junit4]   2> 402289 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
:    [junit4]   2> 402289 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/'
:    [junit4]   2> 402303 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/solr.xml
:    [junit4]   2> 402308 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoresLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores
:    [junit4]   2> 402308 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer New CoreContainer 1809701273
:    [junit4]   2> 402309 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/]
:    [junit4]   2> 402309 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer loading shared library: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/lib
:    [junit4]   2> 402309 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/lib).
:    [junit4]   2> 402315 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
:    [junit4]   2> 402317 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
:    [junit4]   2> 402317 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
:    [junit4]   2> 402317 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
:    [junit4]   2> 402317 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38050/solr
:    [junit4]   2> 402318 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkController zkHost includes chroot
:    [junit4]   2> 402340 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60774_xk%2Ft
:    [junit4]   2> 402353 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer Overseer (id=null) closing
:    [junit4]   2> 402354 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60774_xk%2Ft
:    [junit4]   2> 402355 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer Overseer (id=93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) starting
:    [junit4]   2> 402363 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
:    [junit4]   2> 402363 INFO  (OverseerCollectionProcessor-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.OverseerCollectionProcessor Process current queue of collection creations
:    [junit4]   2> 402364 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer Starting to work on the main queue
:    [junit4]   2> 402366 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoreContainer No authentication plugin used.
:    [junit4]   2> 402367 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
:    [junit4]   2> 402367 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoresLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores
:    [junit4]   2> 402368 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.SolrCore Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/, coreNodeName=, dataDir=data/,
  shard=}
:    [junit4]   2> 402368 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoresLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/
:    [junit4]   2> 402369 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoresLocator Found 1 core definitions
:    [junit4]   2> 402369 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController publishing state=down
:    [junit4]   2> 402369 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
:    [junit4]   2> 402369 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1
:    [junit4]   2> 402370 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
:    [junit4]   2> 402370 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
:    [junit4]   2> 402370 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController look for our core node name
:    [junit4]   2> 402371 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
:    [junit4]   2>   "core":"collection1",
:    [junit4]   2>   "roles":null,
:    [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
:    [junit4]   2>   "node_name":"127.0.0.1:60774_xk%2Ft",
:    [junit4]   2>   "numShards":"1",
:    [junit4]   2>   "state":"down",
:    [junit4]   2>   "shard":null,
:    [junit4]   2>   "collection":"control_collection",
:    [junit4]   2>   "operation":"state"} current state version: 0
:    [junit4]   2> 402371 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
:    [junit4]   2>   "core":"collection1",
:    [junit4]   2>   "roles":null,
:    [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
:    [junit4]   2>   "node_name":"127.0.0.1:60774_xk%2Ft",
:    [junit4]   2>   "numShards":"1",
:    [junit4]   2>   "state":"down",
:    [junit4]   2>   "shard":null,
:    [junit4]   2>   "collection":"control_collection",
:    [junit4]   2>   "operation":"state"}
:    [junit4]   2> 402371 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
:    [junit4]   2> 402372 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
:    [junit4]   2> 403370 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
:    [junit4]   2> 403371 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:control_collection
:    [junit4]   2> 403371 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController Collection zkNode exists
:    [junit4]   2> 403372 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/'
:    [junit4]   2> 403384 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
:    [junit4]   2> 403393 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
:    [junit4]   2> 403396 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.0.0
:    [junit4]   2> 403404 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.Config Loaded SolrConfig: solrconfig.xml
:    [junit4]   2> 403405 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.IndexSchema Reading Solr Schema from /configs/conf1/schema.xml
:    [junit4]   2> 403410 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
:    [junit4]   2> 403498 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
:    [junit4]   2> 403504 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
:    [junit4]   2> 403505 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.IndexSchema unique key field: id
:    [junit4]   2> 403514 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
:    [junit4]   2> 403517 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
:    [junit4]   2> 403518 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
:    [junit4]   2> 403519 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
:    [junit4]   2> 403519 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
:    [junit4]   2> 403519 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
:    [junit4]   2> 403520 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
:    [junit4]   2> 403520 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
:    [junit4]   2> 403520 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
:    [junit4]   2> 403520 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
:    [junit4]   2> 403521 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/], dataDir=[null]
:    [junit4]   2> 403521 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c0e52a6
:    [junit4]   2> 403521 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/data
:    [junit4]   2> 403522 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/data/index/
:    [junit4]   2> 403522 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
:    [junit4]   2> 403522 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/data/index
:    [junit4]   2> 403522 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
:    [junit4]   2> 403523 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onCommit: commits: num=1
:    [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@5d3d153a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5fcf7f85),segFN=segments_1,generation=1}
:    [junit4]   2> 403523 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
:    [junit4]   2> 403527 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "nodistrib"
:    [junit4]   2> 403527 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
:    [junit4]   2> 403527 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
:    [junit4]   2> 403527 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
:    [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
:    [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
:    [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
:    [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
:    [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
:    [junit4]   2> 403530 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
:    [junit4]   2> 403531 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
:    [junit4]   2> 403532 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
:    [junit4]   2> 403533 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
:    [junit4]   2> 403536 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
:    [junit4]   2> 403536 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
:    [junit4]   2> 403537 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
:    [junit4]   2> 403537 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
:    [junit4]   2> 403538 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Hard AutoCommit: disabled
:    [junit4]   2> 403538 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Soft AutoCommit: disabled
:    [junit4]   2> 403538 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1596911514, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
:    [junit4]   2> 403589 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onInit: commits: num=1
:    [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@5d3d153a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5fcf7f85),segFN=segments_1,generation=1}
:    [junit4]   2> 403589 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
:    [junit4]   2> 403589 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening Searcher@449991a3[collection1] main
:    [junit4]   2> 403590 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
:    [junit4]   2> 403591 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
:    [junit4]   2> 403591 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
:    [junit4]   2> 403591 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
:    [junit4]   2> 403591 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
:    [junit4]   2> 403592 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
:    [junit4]   2> 403592 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
:    [junit4]   2> 403592 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
:    [junit4]   2> 403592 INFO  (searcherExecutor-925-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@449991a3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
:    [junit4]   2> 403593 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog On first searcher opened, looking up max value of version field
:    [junit4]   2> 403593 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
:    [junit4]   2> 403593 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
:    [junit4]   2> 403593 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1502493507986653184
:    [junit4]   2> 403595 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 2 ms to seed version buckets with highest version 1502493507986653184
:    [junit4]   2> 403595 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController watch zkdir /configs/conf1
:    [junit4]   2> 403596 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CoreContainer registering core: collection1
:    [junit4]   2> 403596 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController Register replica - core:collection1 address:http://127.0.0.1:60774/xk/t collection:control_collection shard:shard1
:    [junit4]   2> 403600 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Running the leader process for shard shard1
:    [junit4]   2> 403602 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
:    [junit4]   2> 403602 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
:    [junit4]   2> 403602 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
:    [junit4]   2> 403602 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60774/xk/t/collection1/
:    [junit4]   2> 403602 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
:    [junit4]   2>   "operation":"leader",
:    [junit4]   2>   "shard":"shard1",
:    [junit4]   2>   "collection":"control_collection"} current state version: 1
:    [junit4]   2> 403603 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
:    [junit4]   2> 403603 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60774/xk/t/collection1/ has no replicas
:    [junit4]   2> 403603 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60774/xk/t/collection1/ shard1
:    [junit4]   2> 403606 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
:    [junit4]   2> 403607 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
:    [junit4]   2>   "operation":"leader",
:    [junit4]   2>   "shard":"shard1",
:    [junit4]   2>   "collection":"control_collection",
:    [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
:    [junit4]   2>   "core":"collection1",
:    [junit4]   2>   "state":"active"} current state version: 1
:    [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController We are http://127.0.0.1:60774/xk/t/collection1/ and leader is http://127.0.0.1:60774/xk/t/collection1/
:    [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60774/xk/t
:    [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
:    [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController publishing state=active
:    [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
:    [junit4]   2> 403759 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
:    [junit4]   2> 403760 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
:    [junit4]   2>   "core":"collection1",
:    [junit4]   2>   "core_node_name":"core_node1",
:    [junit4]   2>   "roles":null,
:    [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
:    [junit4]   2>   "node_name":"127.0.0.1:60774_xk%2Ft",
:    [junit4]   2>   "numShards":"1",
:    [junit4]   2>   "state":"active",
:    [junit4]   2>   "shard":"shard1",
:    [junit4]   2>   "collection":"control_collection",
:    [junit4]   2>   "operation":"state"} current state version: 2
:    [junit4]   2> 403760 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
:    [junit4]   2>   "core":"collection1",
:    [junit4]   2>   "core_node_name":"core_node1",
:    [junit4]   2>   "roles":null,
:    [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
:    [junit4]   2>   "node_name":"127.0.0.1:60774_xk%2Ft",
:    [junit4]   2>   "numShards":"1",
:    [junit4]   2>   "state":"active",
:    [junit4]   2>   "shard":"shard1",
:    [junit4]   2>   "collection":"control_collection",
:    [junit4]   2>   "operation":"state"}
:    [junit4]   2> 403876 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
:    [junit4]   2> 404167 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1
:    [junit4]   2> 404168 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001
:    [junit4]   2> 404171 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.Server jetty-9.2.10.v20150310
:    [junit4]   2> 404174 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@28f169fa{/xk/t,null,AVAILABLE}
:    [junit4]   2> 404176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.ServerConnector Started ServerConnector@2b3b8799{HTTP/1.1}{127.0.0.1:52152}
:    [junit4]   2> 404176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.Server Started @407672ms
:    [junit4]   2> 404176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/xk/t, hostPort=52152, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores}
:    [junit4]   2> 404177 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
:    [junit4]   2> 404177 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/'
:    [junit4]   2> 404192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/solr.xml
:    [junit4]   2> 404209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoresLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores
:    [junit4]   2> 404209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer New CoreContainer 1593301048
:    [junit4]   2> 404209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/]
:    [junit4]   2> 404209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer loading shared library: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/lib
:    [junit4]   2> 404210 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/lib).
:    [junit4]   2> 404217 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
:    [junit4]   2> 404218 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
:    [junit4]   2> 404219 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
:    [junit4]   2> 404219 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
:    [junit4]   2> 404219 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38050/solr
:    [junit4]   2> 404219 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkController zkHost includes chroot
:    [junit4]   2> 405243 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52152_xk%2Ft
:    [junit4]   2> 405252 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.Overseer Overseer (id=null) closing
:    [junit4]   2> 405255 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoreContainer No authentication plugin used.
:    [junit4]   2> 405255 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
:    [junit4]   2> 405256 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoresLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores
:    [junit4]   2> 405257 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.SolrCore Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1, collection=collection1, absoluteInstDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=
 }
:    [junit4]   2> 405257 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoresLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/
:    [junit4]   2> 405257 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoresLocator Found 1 core definitions
:    [junit4]   2> 405258 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController publishing state=down
:    [junit4]   2> 405258 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
:    [junit4]   2> 405258 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1
:    [junit4]   2> 405259 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
:    [junit4]   2> 405259 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController look for our core node name
:    [junit4]   2> 405259 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
:    [junit4]   2> 405260 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
:    [junit4]   2>   "core":"collection1",
:    [junit4]   2>   "roles":null,
:    [junit4]   2>   "base_url":"http://127.0.0.1:52152/xk/t",
:    [junit4]   2>   "node_name":"127.0.0.1:52152_xk%2Ft",
:    [junit4]   2>   "numShards":"2",
:    [junit4]   2>   "state":"down",
:    [junit4]   2>   "shard":null,
:    [junit4]   2>   "collection":"collection1",
:    [junit4]   2>   "operation":"state"} current state version: 3
:    [junit4]   2> 405261 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
:    [junit4]   2>   "core":"collection1",
:    [junit4]   2>   "roles":null,
:    [junit4]   2>   "base_url":"http://127.0.0.1:52152/xk/t",
:    [junit4]   2>   "node_name":"127.0.0.1:52152_xk%2Ft",
:    [junit4]   2>   "numShards":"2",
:    [junit4]   2>   "state":"down",
:    [junit4]   2>   "shard":null,
:    [junit4]   2>   "collection":"collection1",
:    [junit4]   2>   "operation":"state"}
:    [junit4]   2> 405261 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
:    [junit4]   2> 405261 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
:    [junit4]   2> 406260 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
:    [junit4]   2> 406261 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController Check for collection zkNode:collection1
:    [junit4]   2> 406262 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController Collection zkNode exists
:    [junit4]   2> 406266 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/'
:    [junit4]   2> 406284 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
:    [junit4]   2> 406293 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
:    [junit4]   2> 406296 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.0.0
:    [junit4]   2> 406305 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.Config Loaded SolrConfig: solrconfig.xml
:    [junit4]   2> 406306 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.IndexSchema Reading Solr Schema from /configs/conf1/schema.xml
:    [junit4]   2> 406311 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
:    [junit4]   2> 406415 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
:    [junit4]   2> 406422 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
:    [junit4]   2> 406423 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.IndexSchema unique key field: id
:    [junit4]   2> 406434 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
:    [junit4]   2> 406443 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
:    [junit4]   2> 406448 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
:    [junit4]   2> 406454 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
:    [junit4]   2> 406455 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
:    [junit4]   2> 406455 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
:    [junit4]   2> 406457 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
:    [junit4]   2> 406459 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
:    [junit4]   2> 406459 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
:    [junit4]   2> 406459 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
:    [junit4]   2> 406462 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/], dataDir=[null]
:    [junit4]   2> 406463 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c0e52a6
:    [junit4]   2> 406464 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/data
:    [junit4]   2> 406467 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/data/index/
:    [junit4]   2> 406467 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
:    [junit4]   2> 406467 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/data/index
:    [junit4]   2> 406468 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
:    [junit4]   2> 406472 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onCommit: commits: num=1
:    [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@564eed24 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@28bbdc9b),segFN=segments_1,generation=1}
:    [junit4]   2> 406472 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
:    [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "nodistrib"
:    [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
:    [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
:    [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
:    [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
:    [junit4]   2> 406503 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
:    [junit4]   2> 406503 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
:    [junit4]   2> 406503 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
:    [junit4]   2> 406503 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
:    [junit4]   2> 406505 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
:    [junit4]   2> 406506 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
:    [junit4]   2> 406507 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
:    [junit4]   2> 406508 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
:    [junit4]   2> 406518 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
:    [junit4]   2> 406518 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
:    [junit4]   2> 406519 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
:    [junit4]   2> 406519 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
:    [junit4]   2> 406520 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Hard AutoCommit: disabled
:    [junit4]   2> 406520 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Soft AutoCommit: disabled
:    [junit4]   2> 406521 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1596911514, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
:    [junit4]   2> 406572 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onInit: commits: num=1
:    [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@564eed24 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@28bbdc9b),segFN=segments_1,generation=1}
:    [junit4]   2> 406572 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
:    [junit4]   2> 406573 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening Searcher@4a9cfa4a[collection1] main
:    [junit4]   2> 406573 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
:    [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
:    [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
:    [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
:    [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
:    [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
:    [junit4]   2> 406575 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
:    [junit4]   2> 406575 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
:    [junit4]   2> 406576 INFO  (searcherExecutor-936-thread-1-processing-n:127.0.0.1:52152_xk%2Ft x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4a9cfa4a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
:    [junit4]   2> 406576 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog On first searcher opened, looking up max value of version field
:    [junit4]   2> 406576 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
:    [junit4]   2> 406576 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
:    [junit4]   2> 406576 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1502493511114555392
:    [junit4]   2> 406578 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 2 ms to seed version buckets with highest version 1502493511114555392
:    [junit4]
: 
: [...truncated too long message...]
: 
: e_node3 x:collection1] o.a.s.c.SocketProxy Closing 13 connections to: http://127.0.0.1:58289/pn_rez/m, target: http://127.0.0.1:34763/pn_rez/m
:    [junit4]   2> 2318424 WARN  (TEST-HttpPartitionTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:39551_pn_rez%2Fm c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SocketProxy Closing 7 connections to: http://127.0.0.1:39551/pn_rez/m, target: http://127.0.0.1:46711/pn_rez/m
:    [junit4]   2> 2318424 WARN  (TEST-HttpPartitionTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:39551_pn_rez%2Fm c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SocketProxy Closing 5 connections to: http://127.0.0.1:44114/pn_rez/m, target: http://127.0.0.1:41855/pn_rez/m
:    [junit4]   2> 2318425 INFO  (SUITE-HttpPartitionTest-seed#[3AF6852C6379681]-worker) [n:127.0.0.1:39551_pn_rez%2Fm c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
:    [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=5063, maxDocsPerChunk=251, blockSize=4), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=5063, blockSize=4)), sim=DefaultSimilarity, locale=en_AU, timezone=America/Recife
:    [junit4]   2> NOTE: Linux 3.13.0-52-generic amd64/Oracle Corporation 1.8.0_25 (64-bit)/cpus=4,threads=1,free=246850776,total=528482304
:    [junit4]   2> NOTE: All tests run in this JVM: [TestCopyFieldCollectionResource, LeaderElectionIntegrationTest, ActionThrottleTest, TestLazyCores, TestCloudInspectUtil, HighlighterTest, TestImplicitCoreProperties, MultiTermTest, SuggesterWFSTTest, TestObjectReleaseTracker, CSVRequestHandlerTest, TestDFRSimilarityFactory, DistributedSpellCheckComponentTest, TestSolrXml, TestUpdate, TestReqParamsAPI, TestOrdValues, TestCursorMarkWithoutUniqueKey, FastVectorHighlighterTest, MergeStrategyTest, TestFieldSortValues, UUIDFieldTest, TermsComponentTest, TestFunctionQuery, TestArbitraryIndexDir, DebugComponentTest, DateRangeFieldTest, SolrIndexConfigTest, VMParamsZkACLAndCredentialsProvidersTest, IndexBasedSpellCheckerTest, MinimalSchemaTest, TestBadConfig, AliasIntegrationTest, TestInitParams, TestNoOpRegenerator, CursorPagingTest, BlockCacheTest, SSLMigrationTest, TestStandardQParsers, DefaultValueUpdateProcessorTest, TestDistributedMissingSort, TestDistributedStatsComponentCardinality,
  TestCSVLoader, TestHighlightDedupGrouping, TestJmxIntegration, SolrCoreTest, TestBinaryField, SimplePostToolTest, HdfsWriteToMultipleCollectionsTest, HdfsRecoverLeaseTest, ClusterStateTest, TestConfig, TestSolr4Spatial2, TestCodecSupport, AnalysisErrorHandlingTest, TermVectorComponentTest, HighlighterConfigTest, CdcrUpdateLogTest, TestRandomFaceting, SimpleMLTQParserTest, SpatialHeatmapFacetsTest, TestStressLucene, TestFaceting, TestIBSimilarityFactory, TestIntervalFaceting, CurrencyFieldXmlFileTest, TestMiniSolrCloudClusterSSL, TestWordDelimiterFilterFactory, TestBlendedInfixSuggestions, TestTrackingShardHandlerFactory, TestQuerySenderListener, RequestHandlersTest, QueryElevationComponentTest, TestSolrConfigHandler, TestJmxMonitoredMap, TestPostingsSolrHighlighter, IndexSchemaTest, TestMissingGroups, TestSolrIndexConfig, TestSolr4Spatial, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, BasicZkTest, RecoveryZkTest, TestReplicationHandler, Te
 stMergePolicyConfig, SampleTest, NumericFieldsTest, TestElisionMultitermQuery, DirectSolrConnectionTest, TestPhraseSuggestions, TestSweetSpotSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, TestFastLRUCache, DateMathParserTest, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, SpellingQueryConverterTest, DOMUtilTest, TestLRUCache, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, FileUtilsTest, CircularListTest, DistributedIntervalFacetingTest, CollectionStateFormat2Test, DeleteInactiveReplicaTest, DeleteShardTest, DistribJoinFromCollectionTest, HttpPartitionTest]
:    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
:    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HttpPartitionTest -Dtests.seed=3AF6852C6379681 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/x1/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_AU -Dtests.timezone=America/Recife -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
:    [junit4] ERROR   0.00s J0 | HttpPartitionTest (suite) <<<
:    [junit4]    > Throwable #1: java.lang.RuntimeException: file handle leaks: [SeekableByteChannel(/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 3AF6852C6379681-001/index-SimpleFSDirectory-011/segments_2)]
:    [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3AF6852C6379681]:0)
:    [junit4]    > 	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
:    [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
:    [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
:    [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
:    [junit4]    > Caused by: java.lang.Exception
:    [junit4]    > 	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
:    [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:82)
:    [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:272)
:    [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:213)
:    [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:241)
:    [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:361)
:    [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:407)
:    [junit4]    > 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:76)
:    [junit4]    > 	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:110)
:    [junit4]    > 	at org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:42)
:    [junit4]    > 	at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:269)
:    [junit4]    > 	at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:488)
:    [junit4]    > 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:278)
:    [junit4]    > 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:265)
:    [junit4]    > 	at org.apache.lucene.store.BaseDirectoryWrapper.close(BaseDirectoryWrapper.java:46)
:    [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:307)
:    [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.closeCacheValue(CachingDirectoryFactory.java:273)
:    [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:203)
:    [junit4]    > 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1254)
:    [junit4]    > 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:311)
:    [junit4]    > 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:198)
:    [junit4]    > 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)
:    [junit4]    > 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)
:    [junit4]    > 	at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)
:    [junit4]    > 	at org.apache.solr.cloud.ZkController.register(ZkController.java:884)
:    [junit4]    > 	at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
:    [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
:    [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
:    [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
:    [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
:    [junit4]    > 	... 1 more
:    [junit4] Completed [404/503] on J0 in 143.50s, 1 test, 1 error <<< FAILURES!
: 
: [...truncated 337 lines...]
: BUILD FAILED
: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:533: The following error occurred while executing this line:
: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:474: The following error occurred while executing this line:
: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:61: The following error occurred while executing this line:
: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/extra-targets.xml:39: The following error occurred while executing this line:
: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build.xml:230: The following error occurred while executing this line:
: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/common-build.xml:512: The following error occurred while executing this line:
: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:1415: The following error occurred while executing this line:
: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:973: There were test failures: 503 suites, 1986 tests, 1 suite-level error, 1 error, 1 failure, 20 ignored (10 assumptions)
: 
: Total time: 148 minutes 7 seconds
: Build step 'Invoke Ant' marked build as failure
: Archiving artifacts
: No artifacts from Lucene-Solr-NightlyTests-trunk #679 to compare, so performing full copy of artifacts
: Recording test results
: Email was triggered for: Failure
: Sending email for trigger: Failure
: 
: 
: 

-Hoss
http://www.lucidworks.com/

[JENKINS] Lucene-Solr-NightlyTests-trunk - Build # 696 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-trunk/696/

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.HttpPartitionTest

Error Message:
file handle leaks: [SeekableByteChannel(/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 3AF6852C6379681-001/index-SimpleFSDirectory-011/segments_2)]

Stack Trace:
java.lang.RuntimeException: file handle leaks: [SeekableByteChannel(/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 3AF6852C6379681-001/index-SimpleFSDirectory-011/segments_2)]
	at __randomizedtesting.SeedInfo.seed([3AF6852C6379681]:0)
	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
	at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:227)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.Exception
	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:82)
	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:272)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:213)
	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:241)
	at java.nio.file.Files.newByteChannel(Files.java:361)
	at java.nio.file.Files.newByteChannel(Files.java:407)
	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:76)
	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:110)
	at org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:42)
	at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:269)
	at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:488)
	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:278)
	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:265)
	at org.apache.lucene.store.BaseDirectoryWrapper.close(BaseDirectoryWrapper.java:46)
	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:307)
	at org.apache.solr.core.CachingDirectoryFactory.closeCacheValue(CachingDirectoryFactory.java:273)
	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:203)
	at org.apache.solr.core.SolrCore.close(SolrCore.java:1254)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:311)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:198)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)
	at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:884)
	at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more


REGRESSION:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping

Error Message:
distrib-chain-explicit expected LogUpdateProcessor in chain due to @RunAllways, but not found: org.apache.solr.update.processor.DistributedUpdateProcessor@638c6e19, org.apache.solr.update.processor.RemoveBlankFieldUpdateProcessorFactory$1@25bbdfcb, org.apache.solr.update.processor.RunUpdateProcessor@473fea05, 

Stack Trace:
java.lang.AssertionError: distrib-chain-explicit expected LogUpdateProcessor in chain due to @RunAllways, but not found: org.apache.solr.update.processor.DistributedUpdateProcessor@638c6e19, org.apache.solr.update.processor.RemoveBlankFieldUpdateProcessorFactory$1@25bbdfcb, org.apache.solr.update.processor.RunUpdateProcessor@473fea05, 
	at __randomizedtesting.SeedInfo.seed([3AF6852C6379681:724B9684B0DCB14D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=3840, name=collection5, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3840, name=collection5, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:44466/xk/t: collection already exists: awholynewstresscollection_collection5_1
	at __randomizedtesting.SeedInfo.seed([3AF6852C6379681]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:560)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:376)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:328)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:856)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:799)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1570)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1591)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:860)




Build Log:
[...truncated 10189 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/init-core-data-001
   [junit4]   2> 401841 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3AF6852C6379681]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 401841 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[3AF6852C6379681]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /xk/t
   [junit4]   2> 401844 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 401845 INFO  (Thread-2385) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 401845 INFO  (Thread-2385) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 401945 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkTestServer start zk server on port:38050
   [junit4]   2> 401959 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 401961 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 401963 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 401965 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 401967 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 401968 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 401970 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 401971 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 401973 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 401974 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 401976 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 402265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1
   [junit4]   2> 402267 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.Server jetty-9.2.10.v20150310
   [junit4]   2> 402269 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2bd76d64{/xk/t,null,AVAILABLE}
   [junit4]   2> 402288 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.ServerConnector Started ServerConnector@19fab22b{HTTP/1.1}{127.0.0.1:60774}
   [junit4]   2> 402288 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.Server Started @405784ms
   [junit4]   2> 402289 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/tempDir-001/control/data, hostContext=/xk/t, hostPort=60774, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores}
   [junit4]   2> 402289 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
   [junit4]   2> 402289 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/'
   [junit4]   2> 402303 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/solr.xml
   [junit4]   2> 402308 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoresLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores
   [junit4]   2> 402308 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer New CoreContainer 1809701273
   [junit4]   2> 402309 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/]
   [junit4]   2> 402309 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer loading shared library: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/lib
   [junit4]   2> 402309 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/lib).
   [junit4]   2> 402315 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 402317 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 402317 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 402317 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 402317 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38050/solr
   [junit4]   2> 402318 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 402340 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60774_xk%2Ft
   [junit4]   2> 402353 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 402354 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60774_xk%2Ft
   [junit4]   2> 402355 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer Overseer (id=93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) starting
   [junit4]   2> 402363 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 402363 INFO  (OverseerCollectionProcessor-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.OverseerCollectionProcessor Process current queue of collection creations
   [junit4]   2> 402364 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 402366 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 402367 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 402367 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoresLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores
   [junit4]   2> 402368 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.SolrCore Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 402368 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoresLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/
   [junit4]   2> 402369 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.CoresLocator Found 1 core definitions
   [junit4]   2> 402369 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 402369 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 402369 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1
   [junit4]   2> 402370 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 402370 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 402370 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 402371 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
   [junit4]   2>   "node_name":"127.0.0.1:60774_xk%2Ft",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 402371 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
   [junit4]   2>   "node_name":"127.0.0.1:60774_xk%2Ft",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 402371 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 402372 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 403370 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 403371 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:control_collection
   [junit4]   2> 403371 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 403372 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/'
   [junit4]   2> 403384 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 403393 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 403396 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 403404 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.Config Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 403405 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.IndexSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 403410 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 403498 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 403504 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 403505 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 403514 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 403517 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 403518 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 403519 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
   [junit4]   2> 403519 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
   [junit4]   2> 403519 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 403520 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
   [junit4]   2> 403520 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
   [junit4]   2> 403520 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 403520 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 403521 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/], dataDir=[null]
   [junit4]   2> 403521 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c0e52a6
   [junit4]   2> 403521 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/data
   [junit4]   2> 403522 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/data/index/
   [junit4]   2> 403522 WARN  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 403522 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/control-001/cores/collection1/data/index
   [junit4]   2> 403522 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 403523 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@5d3d153a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5fcf7f85),segFN=segments_1,generation=1}
   [junit4]   2> 403523 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
   [junit4]   2> 403527 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 403527 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 403527 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 403527 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 403528 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 403530 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 403531 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 403532 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 403533 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 403536 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
   [junit4]   2> 403536 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 403537 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 403537 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 403538 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Hard AutoCommit: disabled
   [junit4]   2> 403538 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Soft AutoCommit: disabled
   [junit4]   2> 403538 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1596911514, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 403589 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@5d3d153a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5fcf7f85),segFN=segments_1,generation=1}
   [junit4]   2> 403589 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
   [junit4]   2> 403589 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening Searcher@449991a3[collection1] main
   [junit4]   2> 403590 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 403591 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 403591 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
   [junit4]   2> 403591 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 403591 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 403592 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 403592 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 403592 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 403592 INFO  (searcherExecutor-925-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@449991a3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 403593 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog On first searcher opened, looking up max value of version field
   [junit4]   2> 403593 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 403593 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 403593 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1502493507986653184
   [junit4]   2> 403595 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 2 ms to seed version buckets with highest version 1502493507986653184
   [junit4]   2> 403595 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController watch zkdir /configs/conf1
   [junit4]   2> 403596 INFO  (coreLoadExecutor-924-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 403596 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController Register replica - core:collection1 address:http://127.0.0.1:60774/xk/t collection:control_collection shard:shard1
   [junit4]   2> 403600 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Running the leader process for shard shard1
   [junit4]   2> 403602 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 403602 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 403602 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 403602 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60774/xk/t/collection1/
   [junit4]   2> 403602 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection"} current state version: 1
   [junit4]   2> 403603 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 403603 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:60774/xk/t/collection1/ has no replicas
   [junit4]   2> 403603 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60774/xk/t/collection1/ shard1
   [junit4]   2> 403606 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 403607 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "state":"active"} current state version: 1
   [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController We are http://127.0.0.1:60774/xk/t/collection1/ and leader is http://127.0.0.1:60774/xk/t/collection1/
   [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60774/xk/t
   [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController publishing state=active
   [junit4]   2> 403757 INFO  (coreZkRegister-918-thread-1-processing-n:127.0.0.1:60774_xk%2Ft x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:60774_xk%2Ft c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 403759 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 403760 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "core_node_name":"core_node1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
   [junit4]   2>   "node_name":"127.0.0.1:60774_xk%2Ft",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 2
   [junit4]   2> 403760 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "core_node_name":"core_node1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:60774/xk/t",
   [junit4]   2>   "node_name":"127.0.0.1:60774_xk%2Ft",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"active",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 403876 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 404167 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1
   [junit4]   2> 404168 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001
   [junit4]   2> 404171 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.Server jetty-9.2.10.v20150310
   [junit4]   2> 404174 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@28f169fa{/xk/t,null,AVAILABLE}
   [junit4]   2> 404176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.ServerConnector Started ServerConnector@2b3b8799{HTTP/1.1}{127.0.0.1:52152}
   [junit4]   2> 404176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.e.j.s.Server Started @407672ms
   [junit4]   2> 404176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/xk/t, hostPort=52152, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores}
   [junit4]   2> 404177 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
   [junit4]   2> 404177 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/'
   [junit4]   2> 404192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/solr.xml
   [junit4]   2> 404209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoresLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores
   [junit4]   2> 404209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer New CoreContainer 1593301048
   [junit4]   2> 404209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/]
   [junit4]   2> 404209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.CoreContainer loading shared library: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/lib
   [junit4]   2> 404210 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/lib).
   [junit4]   2> 404217 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 404218 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 404219 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 404219 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 404219 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38050/solr
   [junit4]   2> 404219 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 405243 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52152_xk%2Ft
   [junit4]   2> 405252 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 405255 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 405255 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 405256 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoresLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores
   [junit4]   2> 405257 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.SolrCore Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1, collection=collection1, absoluteInstDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 405257 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoresLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/
   [junit4]   2> 405257 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.c.CoresLocator Found 1 core definitions
   [junit4]   2> 405258 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 405258 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 405258 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1
   [junit4]   2> 405259 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:52152_xk%2Ft    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 405259 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 405259 INFO  (zkCallback-227-thread-1-processing-n:127.0.0.1:60774_xk%2Ft) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 405260 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:52152/xk/t",
   [junit4]   2>   "node_name":"127.0.0.1:52152_xk%2Ft",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"} current state version: 3
   [junit4]   2> 405261 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:52152/xk/t",
   [junit4]   2>   "node_name":"127.0.0.1:52152_xk%2Ft",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 405261 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 405261 INFO  (OverseerStateUpdate-93905844134674435-127.0.0.1:60774_xk%2Ft-n_0000000000) [n:127.0.0.1:60774_xk%2Ft    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 406260 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 406261 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController Check for collection zkNode:collection1
   [junit4]   2> 406262 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 406266 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/'
   [junit4]   2> 406284 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 406293 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 406296 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 406305 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.Config Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 406306 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.IndexSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 406311 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 406415 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 406422 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 406423 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 406434 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 406443 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 406448 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 406454 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
   [junit4]   2> 406455 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
   [junit4]   2> 406455 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 406457 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
   [junit4]   2> 406459 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
   [junit4]   2> 406459 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 406459 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 406462 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/], dataDir=[null]
   [junit4]   2> 406463 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c0e52a6
   [junit4]   2> 406464 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/data
   [junit4]   2> 406467 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 406467 WARN  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 406467 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest 3AF6852C6379681-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 406468 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 406472 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@564eed24 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@28bbdc9b),segFN=segments_1,generation=1}
   [junit4]   2> 406472 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
   [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 406502 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 406503 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 406503 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 406503 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 406503 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 406505 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 406506 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 406507 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 406508 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 406518 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
   [junit4]   2> 406518 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 406519 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 406519 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 406520 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Hard AutoCommit: disabled
   [junit4]   2> 406520 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Soft AutoCommit: disabled
   [junit4]   2> 406521 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1596911514, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 406572 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@564eed24 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@28bbdc9b),segFN=segments_1,generation=1}
   [junit4]   2> 406572 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
   [junit4]   2> 406573 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening Searcher@4a9cfa4a[collection1] main
   [junit4]   2> 406573 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
   [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 406574 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 406575 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 406575 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 406576 INFO  (searcherExecutor-936-thread-1-processing-n:127.0.0.1:52152_xk%2Ft x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4a9cfa4a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 406576 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog On first searcher opened, looking up max value of version field
   [junit4]   2> 406576 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 406576 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 406576 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1502493511114555392
   [junit4]   2> 406578 INFO  (coreLoadExecutor-935-thread-1-processing-n:127.0.0.1:52152_xk%2Ft) [n:127.0.0.1:52152_xk%2Ft c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 2 ms to seed version buckets with highest version 1502493511114555392
   [junit4]

[...truncated too long message...]

e_node3 x:collection1] o.a.s.c.SocketProxy Closing 13 connections to: http://127.0.0.1:58289/pn_rez/m, target: http://127.0.0.1:34763/pn_rez/m
   [junit4]   2> 2318424 WARN  (TEST-HttpPartitionTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:39551_pn_rez%2Fm c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SocketProxy Closing 7 connections to: http://127.0.0.1:39551/pn_rez/m, target: http://127.0.0.1:46711/pn_rez/m
   [junit4]   2> 2318424 WARN  (TEST-HttpPartitionTest.test-seed#[3AF6852C6379681]) [n:127.0.0.1:39551_pn_rez%2Fm c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SocketProxy Closing 5 connections to: http://127.0.0.1:44114/pn_rez/m, target: http://127.0.0.1:41855/pn_rez/m
   [junit4]   2> 2318425 INFO  (SUITE-HttpPartitionTest-seed#[3AF6852C6379681]-worker) [n:127.0.0.1:39551_pn_rez%2Fm c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=5063, maxDocsPerChunk=251, blockSize=4), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=5063, blockSize=4)), sim=DefaultSimilarity, locale=en_AU, timezone=America/Recife
   [junit4]   2> NOTE: Linux 3.13.0-52-generic amd64/Oracle Corporation 1.8.0_25 (64-bit)/cpus=4,threads=1,free=246850776,total=528482304
   [junit4]   2> NOTE: All tests run in this JVM: [TestCopyFieldCollectionResource, LeaderElectionIntegrationTest, ActionThrottleTest, TestLazyCores, TestCloudInspectUtil, HighlighterTest, TestImplicitCoreProperties, MultiTermTest, SuggesterWFSTTest, TestObjectReleaseTracker, CSVRequestHandlerTest, TestDFRSimilarityFactory, DistributedSpellCheckComponentTest, TestSolrXml, TestUpdate, TestReqParamsAPI, TestOrdValues, TestCursorMarkWithoutUniqueKey, FastVectorHighlighterTest, MergeStrategyTest, TestFieldSortValues, UUIDFieldTest, TermsComponentTest, TestFunctionQuery, TestArbitraryIndexDir, DebugComponentTest, DateRangeFieldTest, SolrIndexConfigTest, VMParamsZkACLAndCredentialsProvidersTest, IndexBasedSpellCheckerTest, MinimalSchemaTest, TestBadConfig, AliasIntegrationTest, TestInitParams, TestNoOpRegenerator, CursorPagingTest, BlockCacheTest, SSLMigrationTest, TestStandardQParsers, DefaultValueUpdateProcessorTest, TestDistributedMissingSort, TestDistributedStatsComponentCardinality, TestCSVLoader, TestHighlightDedupGrouping, TestJmxIntegration, SolrCoreTest, TestBinaryField, SimplePostToolTest, HdfsWriteToMultipleCollectionsTest, HdfsRecoverLeaseTest, ClusterStateTest, TestConfig, TestSolr4Spatial2, TestCodecSupport, AnalysisErrorHandlingTest, TermVectorComponentTest, HighlighterConfigTest, CdcrUpdateLogTest, TestRandomFaceting, SimpleMLTQParserTest, SpatialHeatmapFacetsTest, TestStressLucene, TestFaceting, TestIBSimilarityFactory, TestIntervalFaceting, CurrencyFieldXmlFileTest, TestMiniSolrCloudClusterSSL, TestWordDelimiterFilterFactory, TestBlendedInfixSuggestions, TestTrackingShardHandlerFactory, TestQuerySenderListener, RequestHandlersTest, QueryElevationComponentTest, TestSolrConfigHandler, TestJmxMonitoredMap, TestPostingsSolrHighlighter, IndexSchemaTest, TestMissingGroups, TestSolrIndexConfig, TestSolr4Spatial, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, BasicZkTest, RecoveryZkTest, TestReplicationHandler, TestMergePolicyConfig, SampleTest, NumericFieldsTest, TestElisionMultitermQuery, DirectSolrConnectionTest, TestPhraseSuggestions, TestSweetSpotSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, TestFastLRUCache, DateMathParserTest, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, SpellingQueryConverterTest, DOMUtilTest, TestLRUCache, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, FileUtilsTest, CircularListTest, DistributedIntervalFacetingTest, CollectionStateFormat2Test, DeleteInactiveReplicaTest, DeleteShardTest, DistribJoinFromCollectionTest, HttpPartitionTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HttpPartitionTest -Dtests.seed=3AF6852C6379681 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/x1/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_AU -Dtests.timezone=America/Recife -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | HttpPartitionTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: file handle leaks: [SeekableByteChannel(/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 3AF6852C6379681-001/index-SimpleFSDirectory-011/segments_2)]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3AF6852C6379681]:0)
   [junit4]    > 	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.Exception
   [junit4]    > 	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:82)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:272)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:213)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:241)
   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:361)
   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:407)
   [junit4]    > 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:76)
   [junit4]    > 	at org.apache.lucene.store.Directory.openChecksumInput(Directory.java:110)
   [junit4]    > 	at org.apache.lucene.store.RawDirectoryWrapper.openChecksumInput(RawDirectoryWrapper.java:42)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:269)
   [junit4]    > 	at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:488)
   [junit4]    > 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:278)
   [junit4]    > 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:265)
   [junit4]    > 	at org.apache.lucene.store.BaseDirectoryWrapper.close(BaseDirectoryWrapper.java:46)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:307)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.closeCacheValue(CachingDirectoryFactory.java:273)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:203)
   [junit4]    > 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1254)
   [junit4]    > 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:311)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:198)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:159)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:348)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1070)
   [junit4]    > 	at org.apache.solr.cloud.ZkController.register(ZkController.java:884)
   [junit4]    > 	at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:225)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	... 1 more
   [junit4] Completed [404/503] on J0 in 143.50s, 1 test, 1 error <<< FAILURES!

[...truncated 337 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:533: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:474: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:61: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/extra-targets.xml:39: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build.xml:230: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/common-build.xml:512: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:1415: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:973: There were test failures: 503 suites, 1986 tests, 1 suite-level error, 1 error, 1 failure, 20 ignored (10 assumptions)

Total time: 148 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No artifacts from Lucene-Solr-NightlyTests-trunk #679 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-trunk - Build # 695 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-trunk/695/

12 tests failed.
REGRESSION:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 30000 ms! ClusterState: {   "collection1":{     "replicationFactor":"1",     "shards":{       "shard1":{         "range":"80000000-ffffffff",         "state":"active",         "replicas":{"core_node2":{             "core":"collection1",             "base_url":"http://127.0.0.1:42449/n_fa",             "node_name":"127.0.0.1:42449_n_fa",             "state":"active",             "leader":"true"}}},       "shard2":{         "range":"0-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1",             "base_url":"http://127.0.0.1:58716/n_fa",             "node_name":"127.0.0.1:58716_n_fa",             "state":"active",             "leader":"true"},           "core_node3":{             "core":"collection1",             "base_url":"http://127.0.0.1:32896/n_fa",             "node_name":"127.0.0.1:32896_n_fa",             "state":"active"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "autoCreated":"true"},   "control_collection":{     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"collection1",             "base_url":"http://127.0.0.1:59437/n_fa",             "node_name":"127.0.0.1:59437_n_fa",             "state":"active",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "autoCreated":"true"},   "c8n_1x2":{     "replicationFactor":"2",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"c8n_1x2_shard1_replica2",             "base_url":"http://127.0.0.1:42449/n_fa",             "node_name":"127.0.0.1:42449_n_fa",             "state":"active",             "leader":"true"},           "core_node2":{             "core":"c8n_1x2_shard1_replica1",             "base_url":"http://127.0.0.1:58716/n_fa",             "node_name":"127.0.0.1:58716_n_fa",             "state":"recovering"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 30000 ms! ClusterState: {
  "collection1":{
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"collection1",
            "base_url":"http://127.0.0.1:42449/n_fa",
            "node_name":"127.0.0.1:42449_n_fa",
            "state":"active",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collection1",
            "base_url":"http://127.0.0.1:58716/n_fa",
            "node_name":"127.0.0.1:58716_n_fa",
            "state":"active",
            "leader":"true"},
          "core_node3":{
            "core":"collection1",
            "base_url":"http://127.0.0.1:32896/n_fa",
            "node_name":"127.0.0.1:32896_n_fa",
            "state":"active"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "autoCreated":"true"},
  "control_collection":{
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "core":"collection1",
            "base_url":"http://127.0.0.1:59437/n_fa",
            "node_name":"127.0.0.1:59437_n_fa",
            "state":"active",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "autoCreated":"true"},
  "c8n_1x2":{
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"c8n_1x2_shard1_replica2",
            "base_url":"http://127.0.0.1:42449/n_fa",
            "node_name":"127.0.0.1:42449_n_fa",
            "state":"active",
            "leader":"true"},
          "core_node2":{
            "core":"c8n_1x2_shard1_replica1",
            "base_url":"http://127.0.0.1:58716/n_fa",
            "node_name":"127.0.0.1:58716_n_fa",
            "state":"recovering"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false"}}
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A:26550846FF335482]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1906)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:176)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:99)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestStressRecovery

Error Message:
7 threads leaked from SUITE scope at org.apache.solr.search.TestStressRecovery:     1) Thread[id=76233, name=WRITER4, state=WAITING, group=TGRP-TestStressRecovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)         at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)         at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)    2) Thread[id=76235, name=WRITER6, state=WAITING, group=TGRP-TestStressRecovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)         at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)         at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)    3) Thread[id=76229, name=WRITER0, state=WAITING, group=TGRP-TestStressRecovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)         at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)         at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)    4) Thread[id=76231, name=WRITER2, state=WAITING, group=TGRP-TestStressRecovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)         at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)         at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)    5) Thread[id=76234, name=WRITER5, state=WAITING, group=TGRP-TestStressRecovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)         at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)         at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)    6) Thread[id=76230, name=WRITER1, state=WAITING, group=TGRP-TestStressRecovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)         at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)         at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)    7) Thread[id=76232, name=WRITER3, state=WAITING, group=TGRP-TestStressRecovery]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)         at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)         at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)         at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 7 threads leaked from SUITE scope at org.apache.solr.search.TestStressRecovery: 
   1) Thread[id=76233, name=WRITER4, state=WAITING, group=TGRP-TestStressRecovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
        at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
        at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)
   2) Thread[id=76235, name=WRITER6, state=WAITING, group=TGRP-TestStressRecovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
        at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
        at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)
   3) Thread[id=76229, name=WRITER0, state=WAITING, group=TGRP-TestStressRecovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
        at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
        at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)
   4) Thread[id=76231, name=WRITER2, state=WAITING, group=TGRP-TestStressRecovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
        at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
        at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)
   5) Thread[id=76234, name=WRITER5, state=WAITING, group=TGRP-TestStressRecovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
        at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
        at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)
   6) Thread[id=76230, name=WRITER1, state=WAITING, group=TGRP-TestStressRecovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
        at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
        at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)
   7) Thread[id=76232, name=WRITER3, state=WAITING, group=TGRP-TestStressRecovery]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
        at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
        at org.apache.solr.search.TestStressRecovery$1.run(TestStressRecovery.java:112)
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestStressRecovery

Error Message:
Captured an uncaught exception in thread: Thread[id=76236, name=READER0, state=RUNNABLE, group=TGRP-TestStressRecovery]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=76236, name=READER0, state=RUNNABLE, group=TGRP-TestStressRecovery]
Caused by: java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A]:0)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:331)
Caused by: java.lang.NullPointerException
	at org.apache.solr.util.TestHarness.query(TestHarness.java:320)
	at org.apache.solr.util.TestHarness.query(TestHarness.java:302)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:306)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestStressRecovery

Error Message:
Captured an uncaught exception in thread: Thread[id=76240, name=READER4, state=RUNNABLE, group=TGRP-TestStressRecovery]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=76240, name=READER4, state=RUNNABLE, group=TGRP-TestStressRecovery]
Caused by: java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A]:0)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:331)
Caused by: java.lang.NullPointerException
	at org.apache.solr.util.TestHarness.query(TestHarness.java:320)
	at org.apache.solr.util.TestHarness.query(TestHarness.java:302)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:306)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestStressRecovery

Error Message:
Captured an uncaught exception in thread: Thread[id=76237, name=READER1, state=RUNNABLE, group=TGRP-TestStressRecovery]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=76237, name=READER1, state=RUNNABLE, group=TGRP-TestStressRecovery]
Caused by: java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A]:0)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:331)
Caused by: java.lang.NullPointerException
	at org.apache.solr.util.TestHarness.query(TestHarness.java:320)
	at org.apache.solr.util.TestHarness.query(TestHarness.java:302)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:306)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestStressRecovery

Error Message:
Captured an uncaught exception in thread: Thread[id=76238, name=READER2, state=RUNNABLE, group=TGRP-TestStressRecovery]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=76238, name=READER2, state=RUNNABLE, group=TGRP-TestStressRecovery]
Caused by: java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A]:0)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:331)
Caused by: java.lang.NullPointerException
	at org.apache.solr.util.TestHarness.query(TestHarness.java:320)
	at org.apache.solr.util.TestHarness.query(TestHarness.java:302)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:306)


FAILED:  junit.framework.TestSuite.org.apache.solr.search.TestStressRecovery

Error Message:
Captured an uncaught exception in thread: Thread[id=76239, name=READER3, state=RUNNABLE, group=TGRP-TestStressRecovery]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=76239, name=READER3, state=RUNNABLE, group=TGRP-TestStressRecovery]
Caused by: java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A]:0)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:331)
Caused by: java.lang.NullPointerException
	at org.apache.solr.util.TestHarness.query(TestHarness.java:320)
	at org.apache.solr.util.TestHarness.query(TestHarness.java:302)
	at org.apache.solr.search.TestStressRecovery$2.run(TestStressRecovery.java:306)


FAILED:  org.apache.solr.TestDistributedSearch.test

Error Message:
Captured an uncaught exception in thread: Thread[id=67609, name=Thread-61006, state=RUNNABLE, group=TGRP-TestDistributedSearch]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=67609, name=Thread-61006, state=RUNNABLE, group=TGRP-TestDistributedSearch]
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A:26550846FF335482]:0)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55138/ix_pty/g/collection1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49106/ix_pty/g/collection1
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:560)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$5.run(BaseDistributedSearchTestCase.java:621)


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {   "collection1":{     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1",             "base_url":"http://127.0.0.1:59748",             "node_name":"127.0.0.1:59748_",             "state":"active",             "leader":"true"},           "core_node2":{             "core":"collection1",             "base_url":"http://127.0.0.1:49869",             "node_name":"127.0.0.1:49869_",             "state":"recovering"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "autoCreated":"true"},   "control_collection":{     "replicationFactor":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"collection1",             "base_url":"http://127.0.0.1:45279",             "node_name":"127.0.0.1:45279_",             "state":"active",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "autoCreated":"true"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
  "collection1":{
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collection1",
            "base_url":"http://127.0.0.1:59748",
            "node_name":"127.0.0.1:59748_",
            "state":"active",
            "leader":"true"},
          "core_node2":{
            "core":"collection1",
            "base_url":"http://127.0.0.1:49869",
            "node_name":"127.0.0.1:49869_",
            "state":"recovering"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "autoCreated":"true"},
  "control_collection":{
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "core":"collection1",
            "base_url":"http://127.0.0.1:45279",
            "node_name":"127.0.0.1:45279_",
            "state":"active",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "autoCreated":"true"}}
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A:26550846FF335482]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1906)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:116)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1432817556184,generation=2,filelist=[_1a9.cfe, _1a9.cfs, _1a9.si, _1or.cfe, _1or.cfs, _1or.si, _1os.cfe, _1os.cfs, _1os.si, _1ot.cfe, _1ot.cfs, _1ot.si, _1ov.cfe, _1ov.cfs, _1ov.si, _fh.fdt, _fh.fdx, _fh.fnm, _fh.nvd, _fh.nvm, _fh.si, _fh_Lucene50_0.doc, _fh_Lucene50_0.tim, _fh_Lucene50_0.tip, _uu.cfe, _uu.cfs, _uu.si, segments_2]}]> but was:<[{indexVersion=1432817556184,generation=2,filelist=[_1a9.cfe, _1a9.cfs, _1a9.si, _1or.cfe, _1or.cfs, _1or.si, _1os.cfe, _1os.cfs, _1os.si, _1ot.cfe, _1ot.cfs, _1ot.si, _1ov.cfe, _1ov.cfs, _1ov.si, _fh.fdt, _fh.fdx, _fh.fnm, _fh.nvd, _fh.nvm, _fh.si, _fh_Lucene50_0.doc, _fh_Lucene50_0.tim, _fh_Lucene50_0.tip, _uu.cfe, _uu.cfs, _uu.si, segments_2]}, {indexVersion=1432817556184,generation=3,filelist=[_1a9.cfe, _1a9.cfs, _1a9.si, _1ox.cfe, _1ox.cfs, _1ox.si, _fh.fdt, _fh.fdx, _fh.fnm, _fh.nvd, _fh.nvm, _fh.si, _fh_Lucene50_0.doc, _fh_Lucene50_0.tim, _fh_Lucene50_0.tip, _uu.cfe, _uu.cfs, _uu.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1432817556184,generation=2,filelist=[_1a9.cfe, _1a9.cfs, _1a9.si, _1or.cfe, _1or.cfs, _1or.si, _1os.cfe, _1os.cfs, _1os.si, _1ot.cfe, _1ot.cfs, _1ot.si, _1ov.cfe, _1ov.cfs, _1ov.si, _fh.fdt, _fh.fdx, _fh.fnm, _fh.nvd, _fh.nvm, _fh.si, _fh_Lucene50_0.doc, _fh_Lucene50_0.tim, _fh_Lucene50_0.tip, _uu.cfe, _uu.cfs, _uu.si, segments_2]}]> but was:<[{indexVersion=1432817556184,generation=2,filelist=[_1a9.cfe, _1a9.cfs, _1a9.si, _1or.cfe, _1or.cfs, _1or.si, _1os.cfe, _1os.cfs, _1os.si, _1ot.cfe, _1ot.cfs, _1ot.si, _1ov.cfe, _1ov.cfs, _1ov.si, _fh.fdt, _fh.fdx, _fh.fnm, _fh.nvd, _fh.nvm, _fh.si, _fh_Lucene50_0.doc, _fh_Lucene50_0.tim, _fh_Lucene50_0.tip, _uu.cfe, _uu.cfs, _uu.si, segments_2]}, {indexVersion=1432817556184,generation=3,filelist=[_1a9.cfe, _1a9.cfs, _1a9.si, _1ox.cfe, _1ox.cfs, _1ox.si, _fh.fdt, _fh.fdx, _fh.fnm, _fh.nvd, _fh.nvm, _fh.si, _fh_Lucene50_0.doc, _fh_Lucene50_0.tim, _fh_Lucene50_0.tip, _uu.cfe, _uu.cfs, _uu.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A:8BD62CAC21873779]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1138)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.search.TestStressRecovery.testStressRecovery

Error Message:
java.lang.AssertionError

Stack Trace:
java.util.concurrent.ExecutionException: java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([AE01379C51CF397A:143B5EC1CE278674]:0)
	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.util.concurrent.FutureTask.get(FutureTask.java:206)
	at org.apache.solr.search.TestStressRecovery.testStressRecovery(TestStressRecovery.java:370)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.update.CdcrUpdateLogTest$1.run(CdcrUpdateLogTest.java:444)
	at org.apache.solr.update.UpdateLog$LogReplayer.doReplay(UpdateLog.java:1320)
	at org.apache.solr.update.UpdateLog$LogReplayer.run(UpdateLog.java:1258)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:156)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:51471: Could not find collection : awholynewstresscollection_collection3_0

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51471: Could not find collection : awholynewstresscollection_collection3_0
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:560)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:376)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:328)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1086)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:856)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:799)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.addReplicaTest(CollectionsAPIDistributedZkTest.java:1103)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:182)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10373 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/init-core-data-001
   [junit4]   2> 473685 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[AE01379C51CF397A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 473685 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[AE01379C51CF397A]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 473688 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 473688 INFO  (Thread-3745) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 473689 INFO  (Thread-3745) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 473789 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.ZkTestServer start zk server on port:44197
   [junit4]   2> 473811 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 473814 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 473816 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 473818 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 473820 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 473822 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 473832 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 473834 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 473836 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 473837 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 473839 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 473862 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 473865 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 473867 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf2/enumsConfig.xml
   [junit4]   2> 473869 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 473871 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 473873 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 473874 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 473876 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 473880 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 473881 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 473883 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 473884 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 474253 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1
   [junit4]   2> 474255 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.e.j.s.Server jetty-9.2.10.v20150310
   [junit4]   2> 474257 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4e2273ad{/,null,AVAILABLE}
   [junit4]   2> 474260 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.e.j.s.ServerConnector Started ServerConnector@59d4589b{HTTP/1.1}{127.0.0.1:51471}
   [junit4]   2> 474260 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.e.j.s.Server Started @477632ms
   [junit4]   2> 474260 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/tempDir-001/control/data, hostContext=/, hostPort=51471, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores}
   [junit4]   2> 474260 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
   [junit4]   2> 474261 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/'
   [junit4]   2> 474276 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/solr.xml
   [junit4]   2> 474281 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.CoresLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores
   [junit4]   2> 474282 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.CoreContainer New CoreContainer 720896740
   [junit4]   2> 474282 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/]
   [junit4]   2> 474282 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.CoreContainer loading shared library: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/lib
   [junit4]   2> 474282 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.SolrResourceLoader Can't find (or read) directory to add to classloader: lib (resolved as: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/lib).
   [junit4]   2> 474299 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 474300 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 474301 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 474301 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 474301 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44197/solr
   [junit4]   2> 474301 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 474326 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51471_
   [junit4]   2> 474330 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 474331 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51471_
   [junit4]   2> 474333 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.Overseer Overseer (id=93900990641602565-127.0.0.1:51471_-n_0000000000) starting
   [junit4]   2> 474339 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 474340 INFO  (OverseerCollectionProcessor-93900990641602565-127.0.0.1:51471_-n_0000000000) [n:127.0.0.1:51471_    ] o.a.s.c.OverseerCollectionProcessor Process current queue of collection creations
   [junit4]   2> 474341 INFO  (OverseerStateUpdate-93900990641602565-127.0.0.1:51471_-n_0000000000) [n:127.0.0.1:51471_    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 474352 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 474352 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 474353 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.CoresLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores
   [junit4]   2> 474354 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.SolrCore Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 474354 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.CoresLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1/
   [junit4]   2> 474354 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.c.CoresLocator Found 1 core definitions
   [junit4]   2> 474363 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1
   [junit4]   2> 474363 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 474364 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 474365 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 474364 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[AE01379C51CF397A]) [n:127.0.0.1:51471_    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 474367 INFO  (zkCallback-560-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_    ] o.a.s.c.DistributedQueue NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 474368 INFO  (OverseerStateUpdate-93900990641602565-127.0.0.1:51471_-n_0000000000) [n:127.0.0.1:51471_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:51471",
   [junit4]   2>   "node_name":"127.0.0.1:51471_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 474372 INFO  (OverseerStateUpdate-93900990641602565-127.0.0.1:51471_-n_0000000000) [n:127.0.0.1:51471_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:51471",
   [junit4]   2>   "node_name":"127.0.0.1:51471_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 474372 INFO  (OverseerStateUpdate-93900990641602565-127.0.0.1:51471_-n_0000000000) [n:127.0.0.1:51471_    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 474373 INFO  (OverseerStateUpdate-93900990641602565-127.0.0.1:51471_-n_0000000000) [n:127.0.0.1:51471_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 475365 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 475365 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:control_collection
   [junit4]   2> 475366 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 475366 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1/'
   [junit4]   2> 475380 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 475391 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 475395 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 475405 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.Config Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 475408 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 475413 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 475514 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 475522 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 475524 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 475540 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 475543 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 475545 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 475546 WARN  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
   [junit4]   2> 475546 WARN  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
   [junit4]   2> 475546 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 475547 WARN  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Unknown key IMPORTANT NOTE
   [junit4]   2> 475547 WARN  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Expected key, got STRING
   [junit4]   2> 475547 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 475547 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 475548 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1/], dataDir=[null]
   [junit4]   2> 475548 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@75c01f94
   [junit4]   2> 475548 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1/data
   [junit4]   2> 475549 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1/data/index/
   [junit4]   2> 475549 WARN  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 475549 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest AE01379C51CF397A-001/control-001/cores/collection1/data/index
   [junit4]   2> 475549 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5261611005583715]
   [junit4]   2> 475550 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@61cf61fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9a48bd6),segFN=segments_1,generation=1}
   [junit4]   2> 475550 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
   [junit4]   2> 475554 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 475554 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 475554 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 475555 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 475555 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 475555 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 475555 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 475556 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 475556 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 475558 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 475559 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 475560 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 475561 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 475565 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
   [junit4]   2> 475565 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 475566 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 475566 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 475567 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Hard AutoCommit: disabled
   [junit4]   2> 475567 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Soft AutoCommit: disabled
   [junit4]   2> 475567 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=3, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6849803053981687]
   [junit4]   2> 475568 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@61cf61fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9a48bd6),segFN=segments_1,generation=1}
   [junit4]   2> 475568 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore newest commit generation = 1
   [junit4]   2> 475568 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening Searcher@2ff194ff[collection1] main
   [junit4]   2> 475569 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 475570 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 475570 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
   [junit4]   2> 475570 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 475570 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 475570 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 475570 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 475571 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 475571 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog On first searcher opened, looking up max value of version field
   [junit4]   2> 475571 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 475571 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 475571 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1502415852238012416
   [junit4]   2> 475572 INFO  (searcherExecutor-1093-thread-1-processing-n:127.0.0.1:51471_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2ff194ff[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 475574 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 3 ms to seed version buckets with highest version 1502415852238012416
   [junit4]   2> 475574 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController watch zkdir /configs/conf1
   [junit4]   2> 475575 INFO  (coreLoadExecutor-1092-thread-1-processing-n:127.0.0.1:51471_) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 475576 INFO  (coreZkRegister-1086-thread-1-processing-n:127.0.0.1:51471_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:51471_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController Register replica - core:collection1 address:http://127.0.0.1:51471 collection:control_collection shard:shard1
   [junit4]   2> 475580 INFO  (coreZkRegister-1086-thread-1-processing-n:127.0.0.1:51471_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:514

[...truncated too long message...]

jetty-security-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-server-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-servlet-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-servlets-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-util-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-webapp-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/server/lib/jetty-xml-9.2.10.v20150310.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/example/example-DIH/solr/db/lib/derby-10.9.1.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/build/core/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/test-framework/lib/junit-4.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/test-framework/lib/randomizedtesting-runner-2.1.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/apacheds-all-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/commons-collections-3.2.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/hadoop-common-2.6.0-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/hadoop-hdfs-2.6.0-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/hadoop-minikdc-2.6.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jcl-over-slf4j-1.7.7.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/build/analysis/icu/lucene-analyzers-icu-6.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/contrib/analysis-extras/lib/icu4j-54.1.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-antlr.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-regexp.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-resolver.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-commons-net.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jmf.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-oro.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-xalan2.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-commons-logging.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jsch.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-junit4.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-javamail.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jai.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jdepend.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-junit.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-bsf.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-swing.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-netrexx.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-bcel.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-testutil.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-log4j.jar:/x1/jenkins/jenkins-slave/tools/hudson.model.JDK/latest1.8/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.1.13.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/temp/junit4-J2-20150528_080851_842.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build/solr-core/test/temp/junit4-J2-20150528_080851_842.suites
   [junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 129.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1484)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:133)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:964)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:961)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:533: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:474: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:61: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/extra-targets.xml:39: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build.xml:230: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/common-build.xml:512: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:1415: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:973: At least one slave process threw an exception, first: Forked process returned with error code: 129.

Total time: 331 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No artifacts from Lucene-Solr-NightlyTests-trunk #679 to compare, so performing full copy of artifacts
ERROR: Failed to archive artifacts: **/*.events,heapdumps/**,**/hs_err_pid*,README.txt
java.io.IOException: Failed to extract /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/transfer of 4 files
	at hudson.FilePath.readFromTar(FilePath.java:2101)
	at hudson.FilePath.copyRecursiveTo(FilePath.java:2013)
	at jenkins.model.StandardArtifactManager.archive(StandardArtifactManager.java:61)
	at com.cloudbees.jenkins.plugins.jsync.archiver.JSyncArtifactManager.remoteSync(JSyncArtifactManager.java:114)
	at com.cloudbees.jenkins.plugins.jsync.archiver.JSyncArtifactManager.archive(JSyncArtifactManager.java:68)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:140)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:756)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:720)
	at hudson.model.Build$BuildExecution.post2(Build.java:182)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:669)
	at hudson.model.Run.execute(Run.java:1731)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:88)
	at hudson.model.Executor.run(Executor.java:232)
Caused by: java.io.IOException: unexpected EOF with 512 bytes unread
	at hudson.org.apache.tools.tar.TarInputStream.read(TarInputStream.java:349)
	at java.io.FilterInputStream.read(FilterInputStream.java:107)
	at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:1792)
	at org.apache.commons.io.IOUtils.copyLarge(IOUtils.java:1769)
	at org.apache.commons.io.IOUtils.copy(IOUtils.java:1744)
	at hudson.util.IOUtils.copy(IOUtils.java:40)
	at hudson.FilePath.readFromTar(FilePath.java:2091)
	... 14 more
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure