You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2015/04/18 11:16:51 UTC

[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2968 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2968/

1 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:10961//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:10961//collection1
	at __randomizedtesting.SeedInfo.seed([70257D9BA4FAA1EF:F87142410A06CC17]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:466)
	... 51 more




Build Log:
[...truncated 10400 lines...]
   [junit4] Suite: org.apache.solr.handler.component.DistributedMLTComponentTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/init-core-data-001
   [junit4]   2> 1812520 T6418 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1812520 T6418 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1813905 T6418 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1
   [junit4]   2> 1813908 T6418 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1813913 T6418 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:10949
   [junit4]   2> 1813913 T6418 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores, hostContext=/, hostPort=10949}
   [junit4]   2> 1813913 T6418 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1813914 T6418 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/'
   [junit4]   2> 1813966 T6418 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/solr.xml
   [junit4]   2> 1814031 T6418 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores
   [junit4]   2> 1814032 T6418 oasc.CoreContainer.<init> New CoreContainer 1244923054
   [junit4]   2> 1814032 T6418 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/]
   [junit4]   2> 1814032 T6418 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/lib
   [junit4]   2> 1814033 T6418 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/lib).
   [junit4]   2> 1814043 T6418 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1814052 T6418 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1814053 T6418 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1814053 T6418 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1814053 T6418 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1814055 T6418 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores
   [junit4]   2> 1814057 T6418 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, config=solrconfig.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1/, transient=false, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1, schema=schema.xml, configSet=collection1}
   [junit4]   2> 1814057 T6418 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1/
   [junit4]   2> 1814058 T6418 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1814058 T6428 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1814060 T6428 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1814060 T6428 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1814061 T6428 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1814110 T6428 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1814136 T6428 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1814194 T6428 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1814195 T6428 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1814203 T6428 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1814564 T6428 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1814576 T6428 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1814578 T6428 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1814585 T6428 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1814588 T6428 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1814591 T6428 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1814591 T6428 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1814592 T6428 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1814592 T6428 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1814592 T6428 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1814592 T6428 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1814593 T6428 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1814593 T6428 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1814593 T6428 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1/data/
   [junit4]   2> 1814593 T6428 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3673453f
   [junit4]   2> 1814594 T6428 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1/data
   [junit4]   2> 1814594 T6428 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1/data/index/
   [junit4]   2> 1814594 T6428 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1814594 T6428 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/control/cores/collection1/data/index
   [junit4]   2> 1814595 T6428 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=31.580078125, floorSegmentMB=0.37890625, forceMergeDeletesPctAllowed=1.0443926582469687, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5401628480397264
   [junit4]   2> 1814595 T6428 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@142c9590 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7b0f2b48),segFN=segments_1,generation=1}
   [junit4]   2> 1814596 T6428 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1814600 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1814600 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1814601 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1814601 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1814601 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1814601 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1814601 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1814602 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1814602 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1814602 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1814602 T6428 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1814602 T6428 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1814616 T6428 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1814618 T6428 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1814619 T6428 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1814621 T6428 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1814625 T6428 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1814627 T6428 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: spellCheckWithWordbreak,/get,defaults,/admin/luke,mltrh,/admin/threads,/schema,/update/csv,/search-facet-def,/update,spellCheckCompRH1,spellCheckCompRH_Direct,/admin/system,/admin/properties,lazy,spellCheckCompRH,/admin/segments,mock,/admin/plugins,/search-facet-invariants,/admin/mbeans,/admin/file,dismax,/debug/dump,/admin/ping,/update/json,/replication,/terms,/config,/update/json/docs,standard,tvrh,/admin/logging,/mlt,spellCheckWithWordbreak_Direct
   [junit4]   2> 1814628 T6428 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1814629 T6428 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1814631 T6428 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1814631 T6428 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1814632 T6428 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1814632 T6428 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1814633 T6428 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=64.955078125, floorSegmentMB=0.873046875, forceMergeDeletesPctAllowed=21.378166179464465, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1814633 T6428 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@142c9590 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7b0f2b48),segFN=segments_1,generation=1}
   [junit4]   2> 1814633 T6428 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1814634 T6428 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4c80599d[collection1] main
   [junit4]   2> 1814634 T6428 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1814634 T6428 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1814634 T6428 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1814634 T6428 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1814658 T6428 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1814708 T6428 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1814709 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1814710 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1814710 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1814710 T6428 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1814710 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1814710 T6418 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1814710 T6418 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1814711 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1814711 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1814712 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1814713 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1814713 T6429 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1814714 T6429 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4c80599d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1816301 T6418 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1
   [junit4]   2> 1816304 T6418 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1816306 T6418 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:10958
   [junit4]   2> 1816306 T6418 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores, hostContext=/, hostPort=10958}
   [junit4]   2> 1816307 T6418 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1816307 T6418 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/'
   [junit4]   2> 1816352 T6418 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/solr.xml
   [junit4]   2> 1816404 T6418 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores
   [junit4]   2> 1816404 T6418 oasc.CoreContainer.<init> New CoreContainer 345238389
   [junit4]   2> 1816404 T6418 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/]
   [junit4]   2> 1816405 T6418 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/lib
   [junit4]   2> 1816406 T6418 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/lib).
   [junit4]   2> 1816415 T6418 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1816421 T6418 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1816422 T6418 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1816422 T6418 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1816422 T6418 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1816423 T6418 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores
   [junit4]   2> 1816425 T6418 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, configSet=collection1, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1, name=collection1, dataDir=data/, transient=false, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1/}
   [junit4]   2> 1816425 T6418 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1/
   [junit4]   2> 1816426 T6418 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1816426 T6440 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1816428 T6440 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1816428 T6440 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1816429 T6440 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1816470 T6440 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1816492 T6440 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1816544 T6440 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1816545 T6440 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1816552 T6440 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1816847 T6440 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1816859 T6440 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1816861 T6440 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1816868 T6440 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1816872 T6440 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1816874 T6440 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1816875 T6440 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1816875 T6440 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1816875 T6440 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1816876 T6440 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1816876 T6440 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1816876 T6440 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1816876 T6440 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1816877 T6440 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1/data/
   [junit4]   2> 1816877 T6440 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3673453f
   [junit4]   2> 1816878 T6440 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1/data
   [junit4]   2> 1816878 T6440 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1/data/index/
   [junit4]   2> 1816878 T6440 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1816878 T6440 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard0/cores/collection1/data/index
   [junit4]   2> 1816879 T6440 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=31.580078125, floorSegmentMB=0.37890625, forceMergeDeletesPctAllowed=1.0443926582469687, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5401628480397264
   [junit4]   2> 1816879 T6440 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@575b3b06 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@268033b8),segFN=segments_1,generation=1}
   [junit4]   2> 1816879 T6440 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1816885 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1816885 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1816885 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1816886 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1816886 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1816886 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1816886 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1816886 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1816887 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1816887 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1816887 T6440 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1816887 T6440 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1816902 T6440 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1816903 T6440 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1816905 T6440 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1816906 T6440 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1816912 T6440 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1816915 T6440 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: spellCheckWithWordbreak,/get,defaults,/admin/luke,mltrh,/admin/threads,/schema,/update/csv,/search-facet-def,/update,spellCheckCompRH1,spellCheckCompRH_Direct,/admin/system,/admin/properties,lazy,spellCheckCompRH,/admin/segments,mock,/admin/plugins,/search-facet-invariants,/admin/mbeans,/admin/file,dismax,/debug/dump,/admin/ping,/update/json,/replication,/terms,/config,/update/json/docs,standard,tvrh,/admin/logging,/mlt,spellCheckWithWordbreak_Direct
   [junit4]   2> 1816916 T6440 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1816917 T6440 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1816919 T6440 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1816919 T6440 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1816921 T6440 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1816921 T6440 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1816922 T6440 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=64.955078125, floorSegmentMB=0.873046875, forceMergeDeletesPctAllowed=21.378166179464465, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1816923 T6440 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@575b3b06 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@268033b8),segFN=segments_1,generation=1}
   [junit4]   2> 1816923 T6440 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1816923 T6440 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@23733fe3[collection1] main
   [junit4]   2> 1816924 T6440 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1816924 T6440 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1816924 T6440 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1816924 T6440 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1816952 T6440 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1817023 T6440 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1817023 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1817024 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1817025 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1817025 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1817025 T6440 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1817025 T6418 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1817025 T6418 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1817025 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1817026 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1817027 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1817028 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1817028 T6441 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1817030 T6441 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@23733fe3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1818755 T6418 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1
   [junit4]   2> 1818758 T6418 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1818761 T6418 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:10961
   [junit4]   2> 1818761 T6418 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores, hostPort=10961, hostContext=/}
   [junit4]   2> 1818762 T6418 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1818762 T6418 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/'
   [junit4]   2> 1818812 T6418 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/solr.xml
   [junit4]   2> 1818883 T6418 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores
   [junit4]   2> 1818884 T6418 oasc.CoreContainer.<init> New CoreContainer 650799026
   [junit4]   2> 1818884 T6418 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/]
   [junit4]   2> 1818884 T6418 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/lib
   [junit4]   2> 1818885 T6418 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/lib).
   [junit4]   2> 1818895 T6418 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1818904 T6418 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1818905 T6418 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1818905 T6418 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1818906 T6418 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1818907 T6418 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores
   [junit4]   2> 1818909 T6418 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, transient=false, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1, name=collection1, dataDir=data/, loadOnStartup=true, configSet=collection1}
   [junit4]   2> 1818909 T6418 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1/
   [junit4]   2> 1818909 T6418 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1818910 T6452 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1818912 T6452 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1818913 T6452 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1818913 T6452 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1818962 T6452 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1818987 T6452 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1819051 T6452 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1819052 T6452 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1819061 T6452 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1819369 T6452 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1819383 T6452 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1819385 T6452 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1819393 T6452 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1819397 T6452 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1819400 T6452 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1819400 T6452 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1819401 T6452 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1819401 T6452 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1819401 T6452 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1819401 T6452 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1819402 T6452 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1819402 T6452 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1819402 T6452 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1/data/
   [junit4]   2> 1819402 T6452 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3673453f
   [junit4]   2> 1819404 T6452 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1/data
   [junit4]   2> 1819404 T6452 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1/data/index/
   [junit4]   2> 1819404 T6452 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1819404 T6452 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard1/cores/collection1/data/index
   [junit4]   2> 1819405 T6452 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=31.580078125, floorSegmentMB=0.37890625, forceMergeDeletesPctAllowed=1.0443926582469687, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5401628480397264
   [junit4]   2> 1819405 T6452 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2cb370ee lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@38bb302f),segFN=segments_1,generation=1}
   [junit4]   2> 1819405 T6452 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1819411 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1819411 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1819411 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1819411 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1819412 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1819412 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1819412 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1819412 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1819412 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1819413 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1819413 T6452 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1819413 T6452 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1819433 T6452 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1819436 T6452 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1819438 T6452 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1819439 T6452 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1819445 T6452 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1819448 T6452 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: spellCheckWithWordbreak,/get,defaults,/admin/luke,mltrh,/admin/threads,/schema,/update/csv,/search-facet-def,/update,spellCheckCompRH1,spellCheckCompRH_Direct,/admin/system,/admin/properties,lazy,spellCheckCompRH,/admin/segments,mock,/admin/plugins,/search-facet-invariants,/admin/mbeans,/admin/file,dismax,/debug/dump,/admin/ping,/update/json,/replication,/terms,/config,/update/json/docs,standard,tvrh,/admin/logging,/mlt,spellCheckWithWordbreak_Direct
   [junit4]   2> 1819449 T6452 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1819451 T6452 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1819453 T6452 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1819453 T6452 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1819454 T6452 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1819454 T6452 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1819456 T6452 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=64.955078125, floorSegmentMB=0.873046875, forceMergeDeletesPctAllowed=21.378166179464465, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1819457 T6452 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2cb370ee lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@38bb302f),segFN=segments_1,generation=1}
   [junit4]   2> 1819457 T6452 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1819457 T6452 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3405f998[collection1] main
   [junit4]   2> 1819457 T6452 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1819458 T6452 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1819458 T6452 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1819458 T6452 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1819478 T6452 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1819535 T6452 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1819535 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1819536 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1819536 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1819536 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1819537 T6452 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1819537 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1819537 T6418 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1819537 T6418 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1819538 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1819538 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1819539 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1819539 T6453 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1819541 T6453 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3405f998[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1821049 T6418 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1
   [junit4]   2> 1821051 T6418 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1821055 T6418 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:10966
   [junit4]   2> 1821055 T6418 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=10966, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores}
   [junit4]   2> 1821055 T6418 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1821056 T6418 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/'
   [junit4]   2> 1821095 T6418 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/solr.xml
   [junit4]   2> 1821143 T6418 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores
   [junit4]   2> 1821143 T6418 oasc.CoreContainer.<init> New CoreContainer 1561449548
   [junit4]   2> 1821143 T6418 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/]
   [junit4]   2> 1821144 T6418 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/lib
   [junit4]   2> 1821144 T6418 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/lib).
   [junit4]   2> 1821154 T6418 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 1821162 T6418 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1821164 T6418 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1821164 T6418 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1821164 T6418 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1821166 T6418 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores
   [junit4]   2> 1821168 T6418 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/, configSet=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1, name=collection1, schema=schema.xml, config=solrconfig.xml, dataDir=data/, loadOnStartup=true}
   [junit4]   2> 1821169 T6418 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/
   [junit4]   2> 1821169 T6418 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1821170 T6464 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1821172 T6464 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1821172 T6464 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1821173 T6464 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1821234 T6464 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1821257 T6464 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1821311 T6464 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1821312 T6464 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1821320 T6464 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1821658 T6464 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1821673 T6464 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1821676 T6464 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1821686 T6464 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1821690 T6464 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1821693 T6464 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1821693 T6464 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1821694 T6464 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1821694 T6464 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1821694 T6464 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1821694 T6464 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1821695 T6464 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1821695 T6464 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1821695 T6464 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/data/
   [junit4]   2> 1821695 T6464 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3673453f
   [junit4]   2> 1821696 T6464 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 1821696 T6464 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/data/index/
   [junit4]   2> 1821696 T6464 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1821697 T6464 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 1821697 T6464 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=31.580078125, floorSegmentMB=0.37890625, forceMergeDeletesPctAllowed=1.0443926582469687, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5401628480397264
   [junit4]   2> 1821698 T6464 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@448b4fb9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@161f1dd7),segFN=segments_1,generation=1}
   [junit4]   2> 1821698 T6464 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1821703 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1821703 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1821704 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1821704 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1821704 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1821704 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1821704 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1821705 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1821705 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1821705 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1821705 T6464 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1821705 T6464 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1821724 T6464 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1821726 T6464 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1821728 T6464 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1821729 T6464 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1821735 T6464 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1821738 T6464 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: spellCheckWithWordbreak,/get,defaults,/admin/luke,mltrh,/admin/threads,/schema,/update/csv,/search-facet-def,/update,spellCheckCompRH1,spellCheckCompRH_Direct,/admin/system,/admin/properties,lazy,spellCheckCompRH,/admin/segments,mock,/admin/plugins,/search-facet-invariants,/admin/mbeans,/admin/file,dismax,/debug/dump,/admin/ping,/update/json,/replication,/terms,/config,/update/json/docs,standard,tvrh,/admin/logging,/mlt,spellCheckWithWordbreak_Direct
   [junit4]   2> 1821739 T6464 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1821740 T6464 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1821742 T6464 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1821742 T6464 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1821744 T6464 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1821744 T6464 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1821745 T6464 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=64.955078125, floorSegmentMB=0.873046875, forceMergeDeletesPctAllowed=21.378166179464465, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1821746 T6464 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@448b4fb9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@161f1dd7),segFN=segments_1,generation=1}
   [junit4]   2> 1821746 T6464 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1821746 T6464 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6874a29a[collection1] main
   [junit4]   2> 1821747 T6464 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1821747 T6464 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1821747 T6464 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1821747 T6464 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1821757 T6464 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1821808 T6464 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1821808 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1821809 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1821809 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1821809 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1821809 T6464 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1821810 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1821810 T6418 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1821810 T6418 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1821810 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1821811 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1821812 T6418 oas.SolrTestCaseJ4.setUp ###Starting test
   [junit4]   2> 1821812 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1821813 T6465 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1821815 T6465 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6874a29a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C2714 name=collection1 org.apache.solr.core.SolrCore@55671f88
   [junit4]   2> 1821817 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {deleteByQuery=*:* (-1498778541867663360)} 0 3
   [junit4]   2> ASYNC  NEW_CORE C2715 name=collection1 org.apache.solr.core.SolrCore@1c14e345
   [junit4]   2> 1821821 T6434 c:collection1 C2715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {deleteByQuery=*:* (-1498778541872906240)} 0 2
   [junit4]   2> ASYNC  NEW_CORE C2716 name=collection1 org.apache.solr.core.SolrCore@60b14fb1
   [junit4]   2> 1821826 T6448 c:collection1 C2716 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {deleteByQuery=*:* (-1498778541878149120)} 0 3
   [junit4]   2> ASYNC  NEW_CORE C2717 name=collection1 org.apache.solr.core.SolrCore@5dc82550
   [junit4]   2> 1821830 T6460 c:collection1 C2717 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {deleteByQuery=*:* (-1498778541882343424)} 0 2
   [junit4]   2> 1821834 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[1 (1498778541887586304)]} 0 1
   [junit4]   2> 1821838 T6434 c:collection1 C2715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[1 (1498778541891780608)]} 0 1
   [junit4]   2> 1821840 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[2 (1498778541894926336)]} 0 0
   [junit4]   2> 1821844 T6448 c:collection1 C2716 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[2 (1498778541898072064)]} 0 1
   [junit4]   2> 1821847 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[3 (1498778541902266368)]} 0 0
   [junit4]   2> 1821851 T6460 c:collection1 C2717 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[3 (1498778541905412096)]} 0 1
   [junit4]   2> 1821853 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[4 (1498778541908557824)]} 0 0
   [junit4]   2> 1821856 T6434 c:collection1 C2715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[4 (1498778541911703552)]} 0 0
   [junit4]   2> 1821859 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[5 (1498778541914849280)]} 0 0
   [junit4]   2> 1821862 T6448 c:collection1 C2716 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[5 (1498778541917995008)]} 0 0
   [junit4]   2> 1821865 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[6 (1498778541921140736)]} 0 0
   [junit4]   2> 1821868 T6460 c:collection1 C2717 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[6 (1498778541924286464)]} 0 0
   [junit4]   2> 1821871 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[7 (1498778541927432192)]} 0 0
   [junit4]   2> 1821874 T6434 c:collection1 C2715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[7 (1498778541930577920)]} 0 0
   [junit4]   2> 1821877 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[8 (1498778541933723648)]} 0 0
   [junit4]   2> 1821880 T6448 c:collection1 C2716 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[8 (1498778541936869376)]} 0 0
   [junit4]   2> 1821883 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[9 (1498778541940015104)]} 0 0
   [junit4]   2> 1821886 T6460 c:collection1 C2717 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[9 (1498778541943160832)]} 0 0
   [junit4]   2> 1821889 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[10 (1498778541946306560)]} 0 0
   [junit4]   2> 1821892 T6434 c:collection1 C2715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[10 (1498778541949452288)]} 0 0
   [junit4]   2> 1821896 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[12 (1498778541953646592)]} 0 0
   [junit4]   2> 1821899 T6460 c:collection1 C2717 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[12 (1498778541956792320)]} 0 0
   [junit4]   2> 1821902 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[13 (1498778541959938048)]} 0 0
   [junit4]   2> 1821905 T6434 c:collection1 C2715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[13 (1498778541963083776)]} 0 0
   [junit4]   2> 1821908 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[14 (1498778541966229504)]} 0 0
   [junit4]   2> 1821911 T6448 c:collection1 C2716 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[14 (1498778541969375232)]} 0 0
   [junit4]   2> 1821915 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[15 (1498778541973569536)]} 0 0
   [junit4]   2> 1821918 T6460 c:collection1 C2717 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[15 (1498778541976715264)]} 0 0
   [junit4]   2> 1821921 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[16 (1498778541979860992)]} 0 0
   [junit4]   2> 1821924 T6434 c:collection1 C2715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[16 (1498778541983006720)]} 0 0
   [junit4]   2> 1821929 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[17 (1498778541987201024)]} 0 1
   [junit4]   2> 1821933 T6448 c:collection1 C2716 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[17 (1498778541991395328)]} 0 1
   [junit4]   2> 1821935 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[18 (1498778541994541056)]} 0 0
   [junit4]   2> 1821938 T6460 c:collection1 C2717 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[18 (1498778541997686784)]} 0 0
   [junit4]   2> 1821941 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[19 (1498778542000832512)]} 0 0
   [junit4]   2> 1821945 T6434 c:collection1 C2715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={version=2&wt=javabin} {add=[19 (1498778542003978240)]} 0 1
   [junit4]   2> 1821950 T6425 c:collection1 C2714 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params

[...truncated too long message...]

solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/data/index;done=false>>]
   [junit4]   2> 1912808 T6418 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 1912809 T6418 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedMLTComponentTest -Dtests.method=test -Dtests.seed=70257D9BA4FAA1EF -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_EG -Dtests.timezone=US/Pacific-New -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    101s J3 | DistributedMLTComponentTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:10961//collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([70257D9BA4FAA1EF:F87142410A06CC17]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:466)
   [junit4]    > 	... 51 more
   [junit4]   2> 1913078 T6418 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 70257D9BA4FAA1EF-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {lowerfilt1and2=BlockTreeOrds(blocksize=128), timestamp=BlockTreeOrds(blocksize=128), range_facet_l=BlockTreeOrds(blocksize=128), intDefault=PostingsFormat(name=Memory doPackFST= true), lowerfilt1=Lucene50(blocksize=128), multiDefault=BlockTreeOrds(blocksize=128), lowerfilt=BlockTreeOrds(blocksize=128), id=Lucene50(blocksize=128), _version_=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=ar_EG, timezone=US/Pacific-New
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=150262968,total=330825728
   [junit4]   2> NOTE: All tests run in this JVM: [SimpleMLTQParserTest, SolrCmdDistributorTest, JSONWriterTest, OverseerStatusTest, CustomCollectionTest, TestManagedSchema, TestJsonRequest, TestFunctionQuery, PrimUtilsTest, DateRangeFieldTest, TestLFUCache, TestSuggestSpellingConverter, TestManagedSynonymFilterFactory, TestExactSharedStatsCache, TestHashPartitioner, DistribDocExpirationUpdateProcessorTest, TestImplicitCoreProperties, DistributedIntervalFacetingTest, OverriddenZkACLAndCredentialsProvidersTest, PreAnalyzedFieldTest, FastVectorHighlighterTest, TestConfigReload, PolyFieldTest, QueryResultKeyTest, TestLMJelinekMercerSimilarityFactory, SimpleCollectionCreateDeleteTest, DeleteLastCustomShardedReplicaTest, EchoParamsTest, BasicDistributedZk2Test, StandardRequestHandlerTest, StatsComponentTest, TestDefaultSimilarityFactory, TestExactStatsCache, TestDistributedMissingSort, TestSchemaSimilarityResource, TestSimpleQParserPlugin, TestBlobHandler, TestCursorMarkWithoutUniqueKey, DirectUpdateHandlerOptimizeTest, AsyncMigrateRouteKeyTest, PingRequestHandlerTest, DirectSolrConnectionTest, ParsingFieldUpdateProcessorsTest, SpellCheckCollatorTest, TestMergePolicyConfig, TestReload, RemoteQueryErrorTest, PathHierarchyTokenizerFactoryTest, DistributedQueueTest, ResponseHeaderTest, TestStressLucene, TestLeaderElectionZkExpiry, TestSolrConfigHandlerConcurrent, TestReversedWildcardFilterFactory, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TermVectorComponentTest, QueryElevationComponentTest, TestSort, UUIDFieldTest, ShardSplitTest, TestCloudSchemaless, DeleteInactiveReplicaTest, TestDynamicLoading, TestRebalanceLeaders, TestReplicaProperties, RollingRestartTest, TriLevelCompositeIdRoutingTest, TestManagedSchemaDynamicFieldResource, TestSolrConfigHandler, DistributedTermsComponentTest, DistributedMLTComponentTest]
   [junit4] Completed [358/483] on J3 in 100.59s, 1 test, 1 error <<< FAILURES!

[...truncated 405 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:511: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:991: There were test failures: 483 suites, 1916 tests, 1 error, 64 ignored (23 assumptions)

Total time: 52 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2966
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 228262245 bytes
Compression is 0.0%
Took 48 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2971 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2971/

3 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (30 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (30 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([96710D3423572C68:1E2532EE8DAB4190]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:11827/dd/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:11827/dd/collection1
	at __randomizedtesting.SeedInfo.seed([96710D3423572C68:1E2532EE8DAB4190]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:235)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:227)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:466)
	... 51 more


REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds

Error Message:
1: soft wasn't fast enough

Stack Trace:
java.lang.AssertionError: 1: soft wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([96710D3423572C68:CA64A30DC8D56D10]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:316)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9267 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> log4j:WARN No such property [conversionPattern] in org.apache.solr.util.SolrLogLayout.
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001
   [junit4]   2> 4631 T11 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 4808 T11 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 5279 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 5288 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 5289 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 5290 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 6670 T11 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 7014 T11 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 7687 T11 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 7760 T11 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 7979 T11 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 11848 T11 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 12033 T11 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 12042 T11 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 13477 T11 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 13520 T11 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 13530 T11 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 13542 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 13543 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 13544 T11 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 13546 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 13547 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 13590 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 13591 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 13592 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 13785 T11 oasc.CoreContainer.<init> New CoreContainer 331608321
   [junit4]   2> 13787 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 13788 T11 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 13790 T11 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 13917 T11 oashc.HttpShardHandlerFactory.init created with socketTimeout : 600000,connTimeout : 60000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 14861 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 14892 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 14910 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 14924 T11 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 15552 T11 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, name=collection1, loadOnStartup=true, shard=shard1, schema=schema.xml, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, instanceDir=collection1, transient=false, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001}
   [junit4]   2> 15569 T12 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 15572 T12 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 15573 T12 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 15574 T12 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 15788 T12 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 15842 T12 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 15952 T12 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 15953 T12 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 16032 T12 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 16583 T12 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 16620 T12 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 16625 T12 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 16672 T12 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 16682 T12 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 16691 T12 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 16693 T12 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 16694 T12 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 16695 T12 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 16697 T12 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 16698 T12 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 16707 T12 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 16718 T12 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 16761 T12 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/
   [junit4]   2> 16771 T12 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 17010 T12 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001
   [junit4]   2> 17011 T12 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index/
   [junit4]   2> 17012 T12 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 17044 T12 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index
   [junit4]   2> 17047 T12 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 17393 T12 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_1,generation=1}
   [junit4]   2> 17400 T12 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 19980 T12 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 20181 T12 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 20190 T12 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 20191 T12 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 20191 T12 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 20192 T12 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 20192 T12 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 20251 T12 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 20252 T12 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 20594 T12 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 20596 T12 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 20596 T12 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 21150 T12 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21199 T12 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21203 T12 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21206 T12 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21596 T12 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 21764 T12 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/replication,dismax,/debug/dump,/admin/properties,/schema,standard,/update/json,/update/json/docs,spellCheckWithWordbreak_Direct,/config,mltrh,/admin/logging,spellCheckCompRH,spellCheckCompRH_Direct,/admin/plugins,/get,/admin/file,/terms,/admin/luke,/admin/mbeans,lazy,/update,spellCheckCompRH1,/mlt,mock,defaults,/update/csv,tvrh,/admin/segments,/search-facet-invariants,spellCheckWithWordbreak,/admin/system,/search-facet-def,/admin/ping
   [junit4]   2> 21777 T12 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 21780 T12 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 21965 T12 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 21966 T12 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 22038 T12 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 22039 T12 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 22040 T12 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=41.9423828125, floorSegmentMB=0.90234375, forceMergeDeletesPctAllowed=21.715227634028462, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2565476620380965
   [junit4]   2> 22076 T12 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_1,generation=1}
   [junit4]   2> 22077 T12 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 22384 T12 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@70a21e54[collection1] main
   [junit4]   2> 22489 T12 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 22686 T12 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 22687 T12 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 22958 T12 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 23709 T12 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 24592 T12 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 24594 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 24598 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 24599 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 24600 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 24604 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 24607 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 24609 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 24611 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 24613 T12 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 24613 T13 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 24629 T13 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@70a21e54[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 24650 T11 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 24724 T11 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 24725 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 24727 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 24728 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 24729 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 24909 T11 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 24952 T11 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 25080 T11 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 25083 T11 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 25185 T11 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 26052 T11 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 26101 T11 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 26106 T11 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 26159 T11 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 26168 T11 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 26179 T11 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 26181 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 26182 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 26183 T11 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 26184 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 26185 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 26186 T11 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 26187 T11 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/
   [junit4]   2> 26188 T11 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 26189 T11 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index/
   [junit4]   2> 26195 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 26197 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 26198 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 26199 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 26200 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 26201 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 26202 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 26203 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 26205 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 26207 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 26208 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 26208 T11 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 26244 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26249 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26254 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26258 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26266 T11 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 26272 T11 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/replication,dismax,/debug/dump,/admin/properties,/schema,standard,/update/json,/update/json/docs,spellCheckWithWordbreak_Direct,/config,mltrh,/admin/logging,spellCheckCompRH,spellCheckCompRH_Direct,/admin/plugins,/get,/admin/file,/terms,/admin/luke,/admin/mbeans,lazy,/update,spellCheckCompRH1,/mlt,mock,defaults,/update/csv,tvrh,/admin/segments,/search-facet-invariants,spellCheckWithWordbreak,/admin/system,/search-facet-def,/admin/ping
   [junit4]   2> 26277 T11 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 26283 T11 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 26295 T11 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 26297 T11 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 26298 T11 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@597d9abe[collection1] main
   [junit4]   2> 26300 T11 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 26302 T11 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 26303 T11 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 26304 T11 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 26307 T11 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 26330 T11 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 26332 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 26333 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 26334 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 26334 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 26335 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 26337 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 26339 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 26339 T11 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=98.6572265625, floorSegmentMB=0.421875, forceMergeDeletesPctAllowed=14.483422308151631, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1698424553270576
   [junit4]   2> 26343 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 26344 T11 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_1,generation=1}
   [junit4]   2> 26346 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 26347 T11 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 26348 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 26348 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 26350 T11 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 26351 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 26352 T11 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@50530c6d[collection1] main
   [junit4]   2> 26353 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 26354 T11 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 26355 T15 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@597d9abe[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 26356 T11 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@60ee0b0e
   [junit4]   2> 26356 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 26359 T11 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 26360 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 26361 T11 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 26361 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 26363 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 26365 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 26368 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 26370 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 26372 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 26375 T15 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 26376 T15 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@50530c6d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@dd62e57
   [junit4]   2> 28511 T11 c:collection1 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5000 (1498822003433406464)]} 0 2118
   [junit4]   2> 28602 T11 c:collection1 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5001 (1498822004581597184)]} 0 2
   [junit4]   2> 28605 T11 c:collection1 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5002 (1498822004585791488)]} 0 1
   [junit4]   2> 28608 T11 c:collection1 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5003 (1498822004588937216)]} 0 1
   [junit4]   2> 28612 T11 c:collection1 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5004 (1498822004592082944)]} 0 1
   [junit4]   2> 29005 T17 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 29615 T11 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 29695 T16 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeRapidAdds -Dtests.seed=96710D3423572C68 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=tr_TR -Dtests.timezone=Canada/Eastern -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 5.11s J0 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: 1: soft wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([96710D3423572C68:CA64A30DC8D56D10]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:316)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 29778 T11 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 29779 T11 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 29781 T11 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 29781 T11 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 29782 T11 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 29974 T11 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 30015 T11 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 30094 T11 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 30096 T11 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 30210 T11 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 30214 T17 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3798d967[collection1] main
   [junit4]   2> 30236 T17 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 30236 T15 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3798d967[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5)))}
   [junit4]   2> 30238 T16 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_2,generation=2}
   [junit4]   2> 30239 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 30240 T16 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 30241 T16 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 30706 T11 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 30742 T11 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 30747 T11 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 30786 T11 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 30794 T11 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 30801 T11 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 30803 T11 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 30803 T11 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 30804 T11 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 30805 T11 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 30806 T11 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 30807 T11 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 30808 T11 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/
   [junit4]   2> 30809 T11 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 30810 T11 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index/
   [junit4]   2> 30822 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 30823 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 30824 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 30825 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 30826 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 30827 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 30828 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 30829 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 30830 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 30831 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 30833 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 30833 T11 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 30862 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30866 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30869 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30872 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30877 T11 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 30880 T11 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/replication,dismax,/debug/dump,/admin/properties,/schema,standard,/update/json,/update/json/docs,spellCheckWithWordbreak_Direct,/config,mltrh,/admin/logging,spellCheckCompRH,spellCheckCompRH_Direct,/admin/plugins,/get,/admin/file,/terms,/admin/luke,/admin/mbeans,lazy,/update,spellCheckCompRH1,/mlt,mock,defaults,/update/csv,tvrh,/admin/segments,/search-facet-invariants,spellCheckWithWordbreak,/admin/system,/search-facet-def,/admin/ping
   [junit4]   2> 30883 T11 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 30886 T11 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 30888 T11 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 30888 T11 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 30891 T11 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1ebee38a[collection1] main
   [junit4]   2> 30892 T11 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 30893 T11 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 30893 T11 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 30894 T11 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 30898 T11 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 30915 T11 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 30917 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 30917 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 30918 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 30918 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 30918 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 30919 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 30920 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 30921 T11 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 30921 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 30924 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 30925 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 30926 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 30927 T11 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_2,generation=2}
   [junit4]   2> 30927 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 30928 T11 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 30928 T18 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1ebee38a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5)))}
   [junit4]   2> 30929 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 30938 T11 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3ad932b2[collection1] main
   [junit4]   2> 30940 T11 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 30940 T18 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3ad932b2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5)))}
   [junit4]   2> 30941 T11 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@dd62e57
   [junit4]   2> 30943 T11 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=5,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=339,transaction_logs_total_number=1}
   [junit4]   2> 30944 T11 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@74e0da2d
   [junit4]   2> 30954 T11 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1498822007041556480)]} 0 7
   [junit4]   2> 30957 T11 c:collection1 C1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 31085 T11 c:collection1 C1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_3,generation=3}
   [junit4]   2> 31085 T11 c:collection1 C1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 31087 T11 c:collection1 C1 oass.SolrIndexSearcher.<init> Opening Searcher@7b7a739[collection1] main
   [junit4]   2> 31088 T11 c:collection1 C1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 31088 T18 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b7a739[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5) Uninverting(_1(5.2.0):C1)))}
   [junit4]   2> 31090 T11 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 133
   [junit4]   2> 31097 T11 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {delete=[529 (-1498822007196745728)]} 0 2
   [junit4]   2> 31599 T20 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 31628 T20 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@32e95912[collection1] main
   [junit4]   2> 31630 T20 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 31631 T18 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@32e95912[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5)))}
   [junit4]   2> 31635 T11 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[550 (1498822007758782464)]} 0 4
   [junit4]   2> 32136 T20 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 32156 T20 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4d590460[collection1] main
   [junit4]   2> 32157 T20 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 32158 T18 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4d590460[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5) Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 32299 T19 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 32302 T19 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_4,generation=4}
   [junit4]   2> 32303 T19 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 32304 T19 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 32305 T19 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 34304 T11 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 34315 T11 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 34316 T11 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 34318 T11 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 34318 T11 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 34319 T11 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 34465 T11 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 34508 T11 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 34586 T11 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 34587 T11 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 34629 T11 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 35054 T11 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 35094 T11 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 35097 T11 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 35122 T11 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 35127 T11 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 35132 T11 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 35133 T11 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 35133 T11 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 35134 T11 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 35135 T11 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 35135 T11 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 35136 T11 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 35136 T11 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/
   [junit4]   2> 35137 T11 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 35137 T11 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index/
   [junit4]   2> 35141 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 35141 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 35142 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 35142 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 35143 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 35143 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 35144 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 35144 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 35145 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 35145 T11 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35146 T11 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35147 T11 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 35164 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35166 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35169 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35171 T11 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35175 T11 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 35177 T11 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/replication,dismax,/debug/dump,/admin/properties,/schema,standard,/update/json,/update/json/docs,spellCheckWithWordbreak_Direct,/config,mltrh,/admin/logging,spellCheckCompRH,spellCheckCompRH_Direct,/admin/plugins,/get,/admin/file,/terms,/admin/luke,/admin/mbeans,lazy,/update,spellCheckCompRH1,/mlt,mock,defaults,/update/csv,tvrh,/admin/segments,/search-facet-invariants,spellCheckWithWordbreak,/admin/system,/search-facet-def,/admin/ping
   [junit4]   2> 35179 T11 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 35180 T11 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 35181 T11 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 35181 T11 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 35183 T11 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3d29accb[collection1] main
   [junit4]   2> 35184 T11 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 35184 T11 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 35185 T11 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 35185 T11 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 35188 T11 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 35203 T11 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 35204 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 35204 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 35205 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 35205 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 35205 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 35206 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 35206 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 35206 T11 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=33, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.121147928434372]
   [junit4]   2> 35207 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 35208 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 35210 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 35211 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 35212 T21 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 35212 T11 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_4,generation=4}
   [junit4]   2> 35212 T21 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3d29accb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5) Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 35213 T11 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 35214 T11 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 35223 T11 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@192b2fef[collection1] main
   [junit4]   2> 35224 T11 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 35224 T21 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@192b2fef[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5) Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 35225 T11 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@74e0da2d
   [junit4]   2> 35227 T11 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=633,transaction_logs_total_number=3}
   [junit4]   2> 35227 T11 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C2 name=collection1 org.apache.solr.core.SolrCore@78658dc1
   [junit4]   2> 35233 T11 c:collection1 C2 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1498822011531558912)]} 0 4
   [junit4]   2> 35734 T23 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 35756 T23 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2cc7fc32[collection1] main
   [junit4]   2> 35758 T23 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 35758 T21 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2cc7fc32[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5) Uninverting(_2(5.2.0):C1) Uninverting(_3(5.2.0):C1)))}
   [junit4]   2> 35760 T11 c:collection1 C2 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[530 (1498822012086255616)]} 0 2
   [junit4]   2> 36260 T23 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 36274 T23 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2c37f70c[collection1] main
   [junit4]   2> 36275 T23 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 36275 T21 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2c37f70c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5) Uninverting(_2(5.2.0):C1) Uninverting(_3(5.2.0):C1) Uninverting(_4(5.2.0):C1)))}
   [junit4]   2> 36434 T22 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 36438 T22 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@427f722 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd1db4f),segFN=segments_5,generation=5}
   [junit4]   2> 36438 T22 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 36439 T22 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 36440 T22 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 39641 T11 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 39643 T11 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 39644 T11 c:collection1 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=331608321
   [junit4]   2> 39644 T11 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@78658dc1
   [junit4]   2> 39645 T11 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=3,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=838,transaction_logs_total_number=4}
   [junit4]   2> 39646 T11 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 39646 T11 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 39647 T11 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 39648 T11 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 39651 T11 c:collection1 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 39652 T11 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 39652 T11 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001/index
   [junit4]   2> 39653 T11 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001;done=false>>]
   [junit4]   2> 39653 T11 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001/init-core-data-001
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest 96710D3423572C68-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=PostingsFormat(name=Asserting), _version_=PostingsFormat(name=Asserting), text=Lucene50(blocksize=128), subject=PostingsFormat(name=Memory doPackFST= true), intDefault=PostingsFormat(name=Asserting), id=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Asserting)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=tr_TR, timezone=Canada/Eastern
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=119182200,total=223346688
   [junit4]   2> NOTE: All tests run in this JVM: [SoftAutoCommitTest]
   [junit4] Completed [3/483] on J0 in 40.79s, 3 tests, 1 failure <<< FAILURES!

[...truncated 520 lines...]
   [junit4] Suite: org.apache.solr.handler.component.DistributedMLTComponentTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 96710D3423572C68-001/init-core-data-001
   [junit4]   2> 625850 T5629 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 625850 T5629 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /dd/
   [junit4]   2> 627359 T5629 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 96710D3423572C68-001/tempDir-001/control/cores/collection1
   [junit4]   2> 627363 T5629 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 627367 T5629 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11808
   [junit4]   2> 627367 T5629 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 96710D3423572C68-001/tempDir-001/control/cores, hostContext=/dd, hostPort=11808}
   [junit4]   2> 627367 T5629 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 627368 T5629 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/tes

[...truncated too long message...]

":"http://127.0.0.1:65387/wzy/oa",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"} current state version: 15
   [junit4]   2> 1292985 T6925 C:control_collection S:shard1 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=286,cumulative_deletesById=148,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=33183,transaction_logs_total_number=1}
   [junit4]   2> 1292985 T7039 N:127.0.0.1:65387_wzy%2Foa oasco.ReplicaMutator.updateState Update state numShards=null message={
   [junit4]   2> 	  "node_name":"127.0.0.1:65387_wzy%2Foa",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65387/wzy/oa",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1292985 T6925 C:control_collection S:shard1 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1292986 T6925 C:control_collection S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1292986 T6925 C:control_collection S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1292988 T6925 C:control_collection S:shard1 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1293001 T7038 N:127.0.0.1:65387_wzy%2Foa oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1293006 T6925 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1293006 T6925 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 96710D3423572C68-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 96710D3423572C68-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 1293007 T6925 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 96710D3423572C68-001/shard-2-001/cores/collection1/data
   [junit4]   2> 1293007 T6925 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 96710D3423572C68-001/shard-2-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 96710D3423572C68-001/shard-2-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 1293007 T6925 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 96710D3423572C68-001/shard-2-001/cores/collection1/data/index
   [junit4]   2> 1293008 T6925 C:control_collection S:shard1 c:collection1 oasc.Overseer.close Overseer (id=93676451008675848-127.0.0.1:65387_wzy%2Foa-n_0000000002) closing
   [junit4]   2> 1293008 T7039 N:127.0.0.1:65387_wzy%2Foa oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:65387_wzy%2Foa
   [junit4]   2> 1293010 T7038 N:127.0.0.1:65387_wzy%2Foa oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1293011 T6925 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/wzy/oa,null}
   [junit4]   2> 1293215 T6925 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:65293 65293
   [junit4]   2> 1293317 T7021 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:65293 65293
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.method=test -Dtests.seed=96710D3423572C68 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_CR -Dtests.timezone=America/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  109s J3 | ChaosMonkeyNothingIsSafeTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: There were too many update fails (30 > 20) - we expect it can happen, but shouldn't easily
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([96710D3423572C68:1E2532EE8DAB4190]:0)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1293327 T6925 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 96710D3423572C68-001
   [junit4]   2> 108714 T6924 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=es_CR, timezone=America/Cordoba
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=144169576,total=310378496
   [junit4]   2> NOTE: All tests run in this JVM: [TestStressUserVersions, ExternalCollectionsTest, DefaultValueUpdateProcessorTest, TestSchemaVersionResource, TestDefaultStatsCache, RequiredFieldsTest, DebugComponentTest, MBeansHandlerTest, IndexBasedSpellCheckerTest, IndexSchemaRuntimeFieldTest, CursorMarkTest, TestJmxMonitoredMap, TestMissingGroups, TestPivotHelperCode, TestNonDefinedSimilarityFactory, TestManagedSchemaFieldTypeResource, RollingRestartTest, CoreMergeIndexesAdminHandlerTest, TestSolrConfigHandlerCloud, CloudExitableDirectoryReaderTest, SSLMigrationTest, FileUtilsTest, CacheHeaderTest, TestJettySolrRunner, TestPhraseSuggestions, SimpleFacetsTest, SliceStateTest, SpatialHeatmapFacetsTest, TestValueSourceCache, ZkSolrClientTest, SynonymTokenizerTest, TestManagedResource, TestHighlightDedupGrouping, OverseerCollectionProcessorTest, CachingDirectoryFactoryTest, TestReload, TestDefaultSearchFieldResource, TestReplicationHandler, RequestHandlersTest, TestDefaultSimilarityFactory, DocValuesMultiTest, DistributedSpellCheckComponentTest, ParsingFieldUpdateProcessorsTest, DistributedMLTComponentTest, UUIDFieldTest, ConcurrentDeleteAndCreateCollectionTest, TestIntervalFaceting, HdfsCollectionsAPIDistributedZkTest, HdfsLockFactoryTest, DateRangeFieldTest, TestCollectionAPI, StandardRequestHandlerTest, TestBinaryResponseWriter, BadComponentTest, TermVectorComponentTest, ShardRoutingCustomTest, TestSerializedLuceneMatchVersion, LeaderFailoverAfterPartitionTest, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed [246/483] on J3 in 109.03s, 1 test, 1 failure <<< FAILURES!

[...truncated 753 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:511: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:991: There were test failures: 483 suites, 1917 tests, 1 error, 2 failures, 65 ignored (24 assumptions)

Total time: 52 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2966
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 230536385 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2970 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2970/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 700 from http://127.0.0.1:27141/gy_wq/b/collection1lastClient and got 38 from http://127.0.0.1:27144/gy_wq/b/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 700 from http://127.0.0.1:27141/gy_wq/b/collection1lastClient and got 38 from http://127.0.0.1:27144/gy_wq/b/collection1
	at __randomizedtesting.SeedInfo.seed([3A7E5DEE25199286:B22A62348BE5FF7E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9532 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/init-core-data-001
   [junit4]   2> 261724 T976 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 261724 T976 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /gy_wq/b
   [junit4]   2> 261729 T976 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 261730 T977 oasc.ZkTestServer$2$1.setClientPort client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 261730 T977 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 261830 T976 oasc.ZkTestServer.run start zk server on port:27119
   [junit4]   2> 261831 T976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 261831 T976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 261835 T984 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@498b31ca name:ZooKeeperConnection Watcher:127.0.0.1:27119 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 261835 T976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 261835 T976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 261835 T976 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 261837 T976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 261838 T976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 261839 T987 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b2d3d4a name:ZooKeeperConnection Watcher:127.0.0.1:27119/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 261839 T976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 261840 T976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 261840 T976 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 261842 T976 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 261843 T976 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 261844 T976 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 261845 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 261846 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 261848 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 261848 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 261951 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 261952 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 261953 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 261954 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 261955 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 261956 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 261957 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 261958 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 261959 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 261960 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 261961 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 261961 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 261963 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 261964 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 261976 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 261976 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 261978 T976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 261979 T976 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 263238 T976 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1
   [junit4]   2> 263240 T976 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 263244 T976 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27122
   [junit4]   2> 263244 T976 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores, hostContext=/gy_wq/b, hostPort=27122, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/tempDir-001/control/data}
   [junit4]   2> 263244 T976 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 263245 T976 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/'
   [junit4]   2> 263278 T976 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/solr.xml
   [junit4]   2> 263318 T976 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores
   [junit4]   2> 263319 T976 oasc.CoreContainer.<init> New CoreContainer 404648483
   [junit4]   2> 263319 T976 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/]
   [junit4]   2> 263319 T976 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/lib
   [junit4]   2> 263320 T976 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/lib).
   [junit4]   2> 263329 T976 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 263334 T976 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 263335 T976 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 263335 T976 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 263336 T976 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 263336 T976 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27119/solr
   [junit4]   2> 263336 T976 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 263336 T976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 263337 T976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 263339 T999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7eba2e3c name:ZooKeeperConnection Watcher:127.0.0.1:27119 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 263339 T976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 263340 T976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 263342 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 263343 T1002 N:127.0.0.1:27122_gy_wq%2Fb oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13a56ccb name:ZooKeeperConnection Watcher:127.0.0.1:27119/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 263343 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 263345 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 263347 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 263348 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 263350 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 263351 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 263353 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 263354 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 263355 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 263356 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27122_gy_wq%2Fb
   [junit4]   2> 263357 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27122_gy_wq%2Fb
   [junit4]   2> 263358 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 263359 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 263360 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 263361 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:27122_gy_wq%2Fb
   [junit4]   2> 263361 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 263363 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer.start Overseer (id=93675673554255875-127.0.0.1:27122_gy_wq%2Fb-n_0000000000) starting
   [junit4]   2> 263364 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 263368 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 263369 T1004 N:127.0.0.1:27122_gy_wq%2Fb oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 263369 T976 N:127.0.0.1:27122_gy_wq%2Fb oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 263369 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 263372 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores
   [junit4]   2> 263376 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, shard=, name=collection1, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1, transient=false, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/, schema=schema.xml, collection=control_collection, config=solrconfig.xml}
   [junit4]   2> 263377 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/
   [junit4]   2> 263377 T976 N:127.0.0.1:27122_gy_wq%2Fb oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 263378 T1006 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 263378 T1006 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 263378 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 263378 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 263379 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27122/gy_wq/b",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27122_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 263380 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27122/gy_wq/b",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27122_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 263380 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 263380 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 263381 T1002 N:127.0.0.1:27122_gy_wq%2Fb oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264379 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 264380 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 264380 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 264380 T1006 N:127.0.0.1:27122_gy_wq%2Fb oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 264381 T1006 N:127.0.0.1:27122_gy_wq%2Fb oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 264381 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/'
   [junit4]   2> 264393 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 264395 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 264403 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 264437 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 264485 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 264587 T1006 N:127.0.0.1:27122_gy_wq%2Fb oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 264596 T1006 N:127.0.0.1:27122_gy_wq%2Fb oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 264860 T1006 N:127.0.0.1:27122_gy_wq%2Fb oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 264863 T1006 N:127.0.0.1:27122_gy_wq%2Fb oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 264865 T1006 N:127.0.0.1:27122_gy_wq%2Fb oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 264870 T1006 N:127.0.0.1:27122_gy_wq%2Fb oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 264887 T1006 N:127.0.0.1:27122_gy_wq%2Fb oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 264887 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 264888 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/data/
   [junit4]   2> 264888 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@33d86a0c
   [junit4]   2> 264889 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/data
   [junit4]   2> 264889 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/data/index/
   [junit4]   2> 264890 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 264890 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/control-001/cores/collection1/data/index
   [junit4]   2> 264890 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.850286463545851]
   [junit4]   2> 264891 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@272f592d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1587ad74),segFN=segments_1,generation=1}
   [junit4]   2> 264892 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 264899 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 264899 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 264900 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 264900 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 264900 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 264900 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 264901 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 264901 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 264902 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 264921 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 264924 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 264926 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 264928 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 264935 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/get,/admin/file,/update/csv,/admin/segments,/config,/admin/plugins,/admin/logging,/admin/threads,/update/json/docs,/admin/mbeans,standard,/update,/schema,/admin/system,/update/json,/admin/ping,/admin/properties,/replication
   [junit4]   2> 264937 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 264939 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 264939 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 264942 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 264942 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 264943 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15921540627164782]
   [junit4]   2> 264943 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@272f592d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1587ad74),segFN=segments_1,generation=1}
   [junit4]   2> 264944 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 264944 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3d8b9c50[collection1] main
   [junit4]   2> 264944 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 264945 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 264946 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 264946 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 264946 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 264947 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 264947 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 264948 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 264948 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 264948 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 264949 T1007 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3d8b9c50[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 264950 T1006 N:127.0.0.1:27122_gy_wq%2Fb c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 264950 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27122/gy_wq/b collection:control_collection shard:shard1
   [junit4]   2> 264951 T976 N:127.0.0.1:27122_gy_wq%2Fb oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 264951 T976 N:127.0.0.1:27122_gy_wq%2Fb oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 264951 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 264954 T976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 264954 T976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 264956 T1013 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@647eb21b name:ZooKeeperConnection Watcher:127.0.0.1:27119/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 264956 T976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 264957 T976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 264957 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 264957 T976 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 264960 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 264960 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 264960 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C136 name=collection1 org.apache.solr.core.SolrCore@699d4cf6 url=http://127.0.0.1:27122/gy_wq/b/collection1 node=127.0.0.1:27122_gy_wq%2Fb C136_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:27122/gy_wq/b, state=down, node_name=127.0.0.1:27122_gy_wq%2Fb, core=collection1}
   [junit4]   2> 264960 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 C136 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27122/gy_wq/b/collection1/
   [junit4]   2> 264961 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 264961 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 C136 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 264961 T976 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 264962 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 C136 oasc.SyncStrategy.syncToMe http://127.0.0.1:27122/gy_wq/b/collection1/ has no replicas
   [junit4]   2> 264962 T976 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 264962 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27122/gy_wq/b/collection1/ shard1
   [junit4]   2> 264963 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264963 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 264963 T1002 N:127.0.0.1:27122_gy_wq%2Fb oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264963 T976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 264965 T976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 264967 T1016 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39db3cbb name:ZooKeeperConnection Watcher:127.0.0.1:27119/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 264967 T976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 264967 T976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 264969 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 264971 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27122/gy_wq/b",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 264973 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 264973 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 264974 T1002 N:127.0.0.1:27122_gy_wq%2Fb oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264974 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264977 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 264978 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 264979 T1002 N:127.0.0.1:27122_gy_wq%2Fb oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264979 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 265020 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ZkController.register We are http://127.0.0.1:27122/gy_wq/b/collection1/ and leader is http://127.0.0.1:27122/gy_wq/b/collection1/
   [junit4]   2> 265021 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:27122/gy_wq/b
   [junit4]   2> 265021 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 265021 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 265022 T1010 N:127.0.0.1:27122_gy_wq%2Fb C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 265023 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 265024 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27122/gy_wq/b",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:27122_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 265025 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27122/gy_wq/b",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:27122_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 265128 T1002 N:127.0.0.1:27122_gy_wq%2Fb oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 265128 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 266245 T976 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1
   [junit4]   2> 266247 T976 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001
   [junit4]   2> 266248 T976 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 266250 T976 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27141
   [junit4]   2> 266250 T976 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=27141, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/gy_wq/b, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/tempDir-001/jetty1}
   [junit4]   2> 266251 T976 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 266251 T976 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/'
   [junit4]   2> 266280 T976 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/solr.xml
   [junit4]   2> 266316 T976 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores
   [junit4]   2> 266317 T976 oasc.CoreContainer.<init> New CoreContainer 847239767
   [junit4]   2> 266317 T976 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/]
   [junit4]   2> 266317 T976 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/lib
   [junit4]   2> 266318 T976 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/lib).
   [junit4]   2> 266327 T976 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 266332 T976 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 266333 T976 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 266333 T976 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 266333 T976 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 266333 T976 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27119/solr
   [junit4]   2> 266333 T976 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 266334 T976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 266334 T976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 266336 T1028 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22b2419c name:ZooKeeperConnection Watcher:127.0.0.1:27119 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 266336 T976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 266337 T976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 266339 T976 N:127.0.0.1:27141_gy_wq%2Fb oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 266341 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75404b4c name:ZooKeeperConnection Watcher:127.0.0.1:27119/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 266341 T976 N:127.0.0.1:27141_gy_wq%2Fb oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 266345 T976 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 267349 T976 N:127.0.0.1:27141_gy_wq%2Fb oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27141_gy_wq%2Fb
   [junit4]   2> 267350 T976 N:127.0.0.1:27141_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27141_gy_wq%2Fb
   [junit4]   2> 267353 T976 N:127.0.0.1:27141_gy_wq%2Fb oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 267355 T976 N:127.0.0.1:27141_gy_wq%2Fb oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores
   [junit4]   2> 267356 T976 N:127.0.0.1:27141_gy_wq%2Fb oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, coreNodeName=, name=collection1, config=solrconfig.xml, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1, dataDir=data/, loadOnStartup=true, shard=, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/}
   [junit4]   2> 267356 T976 N:127.0.0.1:27141_gy_wq%2Fb oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/
   [junit4]   2> 267357 T976 N:127.0.0.1:27141_gy_wq%2Fb oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 267358 T1032 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 267359 T1032 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 267360 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 267360 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 267360 T1032 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 267360 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27141/gy_wq/b",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27141_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"} current state version: 5
   [junit4]   2> 267361 T1032 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 267361 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27141/gy_wq/b",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27141_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 267361 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 267361 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 267362 T1032 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 267362 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 267362 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 267363 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 267363 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 268362 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 268363 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 268363 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 268363 T1032 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 268364 T1032 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 268364 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/'
   [junit4]   2> 268376 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 268378 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 268385 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 268400 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 268433 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 268534 T1032 N:127.0.0.1:27141_gy_wq%2Fb oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 268541 T1032 N:127.0.0.1:27141_gy_wq%2Fb oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 268775 T1032 N:127.0.0.1:27141_gy_wq%2Fb oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 268777 T1032 N:127.0.0.1:27141_gy_wq%2Fb oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 268778 T1032 N:127.0.0.1:27141_gy_wq%2Fb oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 268781 T1032 N:127.0.0.1:27141_gy_wq%2Fb oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 268794 T1032 N:127.0.0.1:27141_gy_wq%2Fb oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 268794 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 268795 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 268795 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@33d86a0c
   [junit4]   2> 268795 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/data
   [junit4]   2> 268796 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 268796 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 268796 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 268797 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.850286463545851]
   [junit4]   2> 268797 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3866e65f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b9e83e6),segFN=segments_1,generation=1}
   [junit4]   2> 268797 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 268803 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 268804 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 268804 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 268804 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 268804 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 268804 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 268805 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 268805 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 268805 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 268818 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 268819 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 268821 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 268823 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 268829 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/get,/admin/file,/update/csv,/admin/segments,/config,/admin/plugins,/admin/logging,/admin/threads,/update/json/docs,/admin/mbeans,standard,/update,/schema,/admin/system,/update/json,/admin/ping,/admin/properties,/replication
   [junit4]   2> 268831 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 268832 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 268832 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 268833 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 268833 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 268834 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=26, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15921540627164782]
   [junit4]   2> 268834 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3866e65f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b9e83e6),segFN=segments_1,generation=1}
   [junit4]   2> 268835 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 268835 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3d5b7f5f[collection1] main
   [junit4]   2> 268835 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 268836 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 268836 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 268836 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 268836 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 268837 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 268837 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 268837 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 268837 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 268838 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 268839 T1033 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3d5b7f5f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 268839 T1032 N:127.0.0.1:27141_gy_wq%2Fb c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 268840 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27141/gy_wq/b collection:collection1 shard:shard1
   [junit4]   2> 268840 T976 N:127.0.0.1:27141_gy_wq%2Fb oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 268840 T976 N:127.0.0.1:27141_gy_wq%2Fb oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 268840 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 268843 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 268845 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 268845 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 268845 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C137 name=collection1 org.apache.solr.core.SolrCore@538d41c3 url=http://127.0.0.1:27141/gy_wq/b/collection1 node=127.0.0.1:27141_gy_wq%2Fb C137_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:27141/gy_wq/b, state=down, node_name=127.0.0.1:27141_gy_wq%2Fb, core=collection1}
   [junit4]   2> 268845 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 C137 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27141/gy_wq/b/collection1/
   [junit4]   2> 268845 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 268845 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 C137 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 268846 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 C137 oasc.SyncStrategy.syncToMe http://127.0.0.1:27141/gy_wq/b/collection1/ has no replicas
   [junit4]   2> 268846 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27141/gy_wq/b/collection1/ shard1
   [junit4]   2> 268846 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 268846 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 268847 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 268847 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 268849 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 268849 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27141/gy_wq/b",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 268850 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 268851 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 268851 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 268899 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ZkController.register We are http://127.0.0.1:27141/gy_wq/b/collection1/ and leader is http://127.0.0.1:27141/gy_wq/b/collection1/
   [junit4]   2> 268900 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:27141/gy_wq/b
   [junit4]   2> 268900 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 268900 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 268900 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 268901 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 268902 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27141/gy_wq/b",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:27141_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"} current state version: 5
   [junit4]   2> 268902 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27141/gy_wq/b",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:27141_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 268903 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 268903 T1036 N:127.0.0.1:27141_gy_wq%2Fb C:collection1 S:shard1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 268904 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 268904 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 269968 T976 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1
   [junit4]   2> 269970 T976 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001
   [junit4]   2> 269970 T976 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 269973 T976 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27144
   [junit4]   2> 269973 T976 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=27144, hostContext=/gy_wq/b, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
   [junit4]   2> 269974 T976 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 269974 T976 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/'
   [junit4]   2> 270003 T976 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/solr.xml
   [junit4]   2> 270039 T976 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores
   [junit4]   2> 270040 T976 oasc.CoreContainer.<init> New CoreContainer 1280096131
   [junit4]   2> 270040 T976 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/]
   [junit4]   2> 270040 T976 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/lib
   [junit4]   2> 270041 T976 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/lib).
   [junit4]   2> 270050 T976 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 270054 T976 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 270055 T976 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 270055 T976 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 270055 T976 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 270056 T976 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27119/solr
   [junit4]   2> 270056 T976 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 270056 T976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 270057 T976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 270059 T1048 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e6cf3b name:ZooKeeperConnection Watcher:127.0.0.1:27119 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 270059 T976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 270060 T976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 270062 T976 N:127.0.0.1:27144_gy_wq%2Fb oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 270063 T1051 N:127.0.0.1:27144_gy_wq%2Fb oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c0f66d0 name:ZooKeeperConnection Watcher:127.0.0.1:27119/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 270063 T976 N:127.0.0.1:27144_gy_wq%2Fb oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 270067 T976 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 271071 T976 N:127.0.0.1:27144_gy_wq%2Fb oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27144_gy_wq%2Fb
   [junit4]   2> 271072 T976 N:127.0.0.1:27144_gy_wq%2Fb oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27144_gy_wq%2Fb
   [junit4]   2> 271075 T976 N:127.0.0.1:27144_gy_wq%2Fb oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 271078 T976 N:127.0.0.1:27144_gy_wq%2Fb oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores
   [junit4]   2> 271080 T976 N:127.0.0.1:27144_gy_wq%2Fb oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1/, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1, config=solrconfig.xml, shard=, coreNodeName=, loadOnStartup=true, dataDir=data/, name=collection1}
   [junit4]   2> 271081 T976 N:127.0.0.1:27144_gy_wq%2Fb oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1/
   [junit4]   2> 271081 T976 N:127.0.0.1:27144_gy_wq%2Fb oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 271083 T1052 N:127.0.0.1:27144_gy_wq%2Fb C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 271083 T1052 N:127.0.0.1:27144_gy_wq%2Fb C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 271085 T1002 N:127.0.0.1:27122_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 271085 T1052 N:127.0.0.1:27144_gy_wq%2Fb oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 271085 T1052 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 271085 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27144/gy_wq/b",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27144_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"} current state version: 5
   [junit4]   2> 271086 T1052 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 271086 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27144/gy_wq/b",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27144_gy_wq%2Fb",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 271086 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 271087 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 271087 T1052 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 271087 T1052 N:127.0.0.1:27144_gy_wq%2Fb oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 271088 T1003 N:127.0.0.1:27122_gy_wq%2Fb oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 271088 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 271088 T1051 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 271089 T1031 N:127.0.0.1:27141_gy_wq%2Fb oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 271089 T1051 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 272087 T1052 N:127.0.0.1:27144_gy_wq%2Fb oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 272088 T1052 N:127.0.0.1:27144_gy_wq%2Fb oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 272088 T1052 N:127.0.0.1:27144_gy_wq%2Fb oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 272089 T1052 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 272089 T1052 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 272090 T1052 N:127.0.0.1:27144_gy_wq%2Fb oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J

[...truncated too long message...]

1762209
   [junit4]   2> 429401 T976 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 429402 T976 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 429403 T1104 N:127.0.0.1:27144_gy_wq%2Fb oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 429404 T976 C:control_collection S:shard1 c:collection1 oasc.Overseer.close Overseer (id=93675673554255883-127.0.0.1:27144_gy_wq%2Fb-n_0000000003) closing
   [junit4]   2> 429404 T1106 N:127.0.0.1:27144_gy_wq%2Fb oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:27144_gy_wq%2Fb
   [junit4]   2> 429406 T1104 N:127.0.0.1:27144_gy_wq%2Fb oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 431707 T1096 N:127.0.0.1:27144_gy_wq%2Fb C157 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 431708 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is closed
   [junit4]   2> 431708 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1f66094e
   [junit4]   2> 431709 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 431709 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=63,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=54,cumulative_deletesById=29,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=6356,transaction_logs_total_number=1}
   [junit4]   2> 431709 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 431710 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 431710 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 431711 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 431718 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 431718 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 431719 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1/data
   [junit4]   2> 431719 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 431719 T1096 N:127.0.0.1:27144_gy_wq%2Fb oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001/shard-2-001/cores/collection1/data/index
   [junit4]   2> 431720 T976 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/gy_wq/b,null}
   [junit4]   2> 431924 T976 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:27119 27119
   [junit4]   2> 431982 T977 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:27119 27119
   [junit4]   2> 431983 T977 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard1/election/93675673554255879-core_node1-n_0000000000
   [junit4]   2> 		2	/solr/overseer_elect/election/93675673554255879-127.0.0.1:27141_gy_wq%2Fb-n_0000000001
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=3A7E5DEE25199286 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=th -Dtests.timezone=Australia/Darwin -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  170s J0 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 700 from http://127.0.0.1:27141/gy_wq/b/collection1lastClient and got 38 from http://127.0.0.1:27144/gy_wq/b/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3A7E5DEE25199286:B22A62348BE5FF7E]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 431997 T976 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 3A7E5DEE25199286-001
   [junit4]   2> 170277 T975 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=BlockTreeOrds(blocksize=128), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), a_t=Lucene50(blocksize=128), a_i=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), rnd_b=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{}, sim=DefaultSimilarity, locale=th, timezone=Australia/Darwin
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=26841072,total=255328256
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrDynamicMBean, TestCollapseQParserPlugin, TestBulkSchemaAPI, TestDistributedMissingSort, TestDFRSimilarityFactory, TestTrie, DisMaxRequestHandlerTest, TestExceedMaxTermLength, SpatialHeatmapFacetsTest, TestSearcherReuse, TestRealTimeGet, TestManagedStopFilterFactory, TestEmbeddedSolrServerConstructors, HdfsBasicDistributedZkTest, OverseerRolesTest, RecoveryZkTest]
   [junit4] Completed [81/483] on J0 in 171.07s, 1 test, 1 failure <<< FAILURES!

[...truncated 1287 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:511: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:991: There were test failures: 483 suites, 1917 tests, 1 failure, 65 ignored (24 assumptions)

Total time: 52 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2966
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 226272439 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2969 - Still Failing

Posted by Timothy Potter <th...@gmail.com>.
I've BadApple'd the RecoveryAfterSoftCommit test as there's something
wrong with it ... there's no way replicating an index with 115 docs
should take >30 seconds ... so seems like left-over cruft or something
because the test runs fine locally (20 beasts)

On Sat, Apr 18, 2015 at 7:17 AM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2969/
>
> 2 tests failed.
> REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test
>
> Error Message:
> There were too many update fails (25 > 20) - we expect it can happen, but shouldn't easily
>
> Stack Trace:
> java.lang.AssertionError: There were too many update fails (25 > 20) - we expect it can happen, but shouldn't easily
>         at __randomizedtesting.SeedInfo.seed([74BED4029EC96EE2:FCEAEBD83035031A]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.assertTrue(Assert.java:43)
>         at org.junit.Assert.assertFalse(Assert.java:68)
>         at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test
>
> Error Message:
> Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {   "control_collection":{     "autoCreated":"true",     "replicationFactor":"1",     "autoAddReplicas":"false",     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"collection1",             "state":"active",             "node_name":"127.0.0.1:63258_ecb%2Fay",             "base_url":"http://127.0.0.1:63258/ecb/ay",             "leader":"true"}}}}},   "collection1":{     "autoCreated":"true",     "replicationFactor":"1",     "autoAddReplicas":"false",     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1",             "state":"active",             "node_name":"127.0.0.1:63266_ecb%2Fay",             "base_url":"http://127.0.0.1:63266/ecb/ay",             "leader":"true"},           "core_node2":{             "core":"collection1",             "state":"recovering",             "node_name":"127.0.0.1:63274_ecb%2Fay",             "base_url":"http://127.0.0.1:63274/ecb/ay"}}}}}}
>
> Stack Trace:
> java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
>   "control_collection":{
>     "autoCreated":"true",
>     "replicationFactor":"1",
>     "autoAddReplicas":"false",
>     "router":{"name":"compositeId"},
>     "maxShardsPerNode":"1",
>     "shards":{"shard1":{
>         "range":"80000000-7fffffff",
>         "state":"active",
>         "replicas":{"core_node1":{
>             "core":"collection1",
>             "state":"active",
>             "node_name":"127.0.0.1:63258_ecb%2Fay",
>             "base_url":"http://127.0.0.1:63258/ecb/ay",
>             "leader":"true"}}}}},
>   "collection1":{
>     "autoCreated":"true",
>     "replicationFactor":"1",
>     "autoAddReplicas":"false",
>     "router":{"name":"compositeId"},
>     "maxShardsPerNode":"1",
>     "shards":{"shard1":{
>         "range":"80000000-7fffffff",
>         "state":"active",
>         "replicas":{
>           "core_node1":{
>             "core":"collection1",
>             "state":"active",
>             "node_name":"127.0.0.1:63266_ecb%2Fay",
>             "base_url":"http://127.0.0.1:63266/ecb/ay",
>             "leader":"true"},
>           "core_node2":{
>             "core":"collection1",
>             "state":"recovering",
>             "node_name":"127.0.0.1:63274_ecb%2Fay",
>             "base_url":"http://127.0.0.1:63274/ecb/ay"}}}}}}
>         at __randomizedtesting.SeedInfo.seed([74BED4029EC96EE2:FCEAEBD83035031A]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
>         at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 9982 lines...]
>    [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
>    [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/init-core-data-001
>    [junit4]   2> 749637 T2557 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
>    [junit4]   2> 749643 T2557 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>    [junit4]   2> 749643 T2558 oasc.ZkTestServer$2$1.setClientPort client port:0.0.0.0/0.0.0.0:0
>    [junit4]   2> 749643 T2558 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
>    [junit4]   2> 749744 T2557 oasc.ZkTestServer.run start zk server on port:42322
>    [junit4]   2> 749745 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 749746 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 749749 T2565 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@653864d2 name:ZooKeeperConnection Watcher:127.0.0.1:42322 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 749750 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 749750 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
>    [junit4]   2> 749751 T2557 oascc.SolrZkClient.makePath makePath: /solr
>    [junit4]   2> 749754 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 749755 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 749756 T2568 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c2e72e name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 749757 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 749757 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
>    [junit4]   2> 749758 T2557 oascc.SolrZkClient.makePath makePath: /collections/collection1
>    [junit4]   2> 749760 T2557 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
>    [junit4]   2> 749761 T2557 oascc.SolrZkClient.makePath makePath: /collections/control_collection
>    [junit4]   2> 749762 T2557 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
>    [junit4]   2> 749763 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
>    [junit4]   2> 749764 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
>    [junit4]   2> 749766 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
>    [junit4]   2> 749767 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
>    [junit4]   2> 749869 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 749870 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 749871 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
>    [junit4]   2> 749872 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
>    [junit4]   2> 749873 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
>    [junit4]   2> 749873 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
>    [junit4]   2> 749874 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
>    [junit4]   2> 749875 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
>    [junit4]   2> 749876 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
>    [junit4]   2> 749876 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
>    [junit4]   2> 749877 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 749878 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 749879 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 749879 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 749880 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
>    [junit4]   2> 749881 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
>    [junit4]   2> 749882 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
>    [junit4]   2> 749882 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
>    [junit4]   2> 750801 T2557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1
>    [junit4]   2> 750804 T2557 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 750809 T2557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42328
>    [junit4]   2> 750809 T2557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=42328, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/tempDir-001/control/data, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores}
>    [junit4]   2> 750809 T2557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
>    [junit4]   2> 750810 T2557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/'
>    [junit4]   2> 750839 T2557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/solr.xml
>    [junit4]   2> 750874 T2557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores
>    [junit4]   2> 750875 T2557 oasc.CoreContainer.<init> New CoreContainer 616784085
>    [junit4]   2> 750875 T2557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/]
>    [junit4]   2> 750875 T2557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/lib
>    [junit4]   2> 750876 T2557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/lib).
>    [junit4]   2> 750885 T2557 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
>    [junit4]   2> 750889 T2557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
>    [junit4]   2> 750890 T2557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 750890 T2557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 750890 T2557 oasc.CoreContainer.load Node Name: 127.0.0.1
>    [junit4]   2> 750891 T2557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42322/solr
>    [junit4]   2> 750891 T2557 oasc.ZkController.checkChrootPath zkHost includes chroot
>    [junit4]   2> 750891 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 750892 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 750893 T2580 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d64d521 name:ZooKeeperConnection Watcher:127.0.0.1:42322 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 750894 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 750895 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
>    [junit4]   2> 750897 T2557 N:127.0.0.1:42328_ oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 750898 T2583 N:127.0.0.1:42328_ oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5449a446 name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 750898 T2557 N:127.0.0.1:42328_ oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 750899 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/queue
>    [junit4]   2> 750901 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
>    [junit4]   2> 750902 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
>    [junit4]   2> 750904 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
>    [junit4]   2> 750905 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
>    [junit4]   2> 750907 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /live_nodes
>    [junit4]   2> 750908 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /aliases.json
>    [junit4]   2> 750909 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /clusterstate.json
>    [junit4]   2> 750910 T2557 N:127.0.0.1:42328_ oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42328_
>    [junit4]   2> 750910 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42328_
>    [junit4]   2> 750911 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer_elect
>    [junit4]   2> 750912 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer_elect/election
>    [junit4]   2> 750913 T2557 N:127.0.0.1:42328_ oasc.Overseer.close Overseer (id=null) closing
>    [junit4]   2> 750914 T2557 N:127.0.0.1:42328_ oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:42328_
>    [junit4]   2> 750915 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
>    [junit4]   2> 750916 T2557 N:127.0.0.1:42328_ oasc.Overseer.start Overseer (id=93674533455265795-127.0.0.1:42328_-n_0000000000) starting
>    [junit4]   2> 750917 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/queue-work
>    [junit4]   2> 750921 T2557 N:127.0.0.1:42328_ oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
>    [junit4]   2> 750922 T2585 N:127.0.0.1:42328_ oasc.OverseerCollectionProcessor.run Process current queue of collection creations
>    [junit4]   2> 750922 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
>    [junit4]   2> 750922 T2557 N:127.0.0.1:42328_ oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
>    [junit4]   2> 750925 T2557 N:127.0.0.1:42328_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores
>    [junit4]   2> 750928 T2557 N:127.0.0.1:42328_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, schema=schema.xml, coreNodeName=, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/, collection=control_collection, dataDir=data/, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1, loadOnStartup=true, config=solrconfig.xml}
>    [junit4]   2> 750928 T2557 N:127.0.0.1:42328_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/
>    [junit4]   2> 750929 T2557 N:127.0.0.1:42328_ oasc.CorePropertiesLocator.discover Found 1 core definitions
>    [junit4]   2> 750929 T2587 N:127.0.0.1:42328_ C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
>    [junit4]   2> 750929 T2587 N:127.0.0.1:42328_ C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 750930 T2587 N:127.0.0.1:42328_ oasc.ZkController.waitForCoreNodeName look for our core node name
>    [junit4]   2> 750930 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 750931 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42328_",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42328"} current state version: 0
>    [junit4]   2> 750931 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42328_",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42328"}
>    [junit4]   2> 750931 T2584 N:127.0.0.1:42328_ oasco.ClusterStateMutator.createCollection building a new cName: control_collection
>    [junit4]   2> 750932 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
>    [junit4]   2> 750933 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 751930 T2587 N:127.0.0.1:42328_ oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
>    [junit4]   2> 751931 T2587 N:127.0.0.1:42328_ oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
>    [junit4]   2> 751931 T2587 N:127.0.0.1:42328_ oasc.ZkController.createCollectionZkNode Collection zkNode exists
>    [junit4]   2> 751932 T2587 N:127.0.0.1:42328_ oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
>    [junit4]   2> 751932 T2587 N:127.0.0.1:42328_ oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 751932 T2587 N:127.0.0.1:42328_ oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/'
>    [junit4]   2> 751947 T2587 N:127.0.0.1:42328_ oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
>    [junit4]   2> 751949 T2587 N:127.0.0.1:42328_ oasc.Config.<init> loaded config solrconfig.xml with version 0
>    [junit4]   2> 751957 T2587 N:127.0.0.1:42328_ oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
>    [junit4]   2> 751974 T2587 N:127.0.0.1:42328_ oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
>    [junit4]   2> 752022 T2587 N:127.0.0.1:42328_ oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>    [junit4]   2> 752124 T2587 N:127.0.0.1:42328_ oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
>    [junit4]   2> 752132 T2587 N:127.0.0.1:42328_ oass.IndexSchema.readSchema [collection1] Schema name=test
>    [junit4]   2> 752405 T2587 N:127.0.0.1:42328_ oass.IndexSchema.readSchema default search field in schema is text
>    [junit4]   2> 752407 T2587 N:127.0.0.1:42328_ oass.IndexSchema.readSchema unique key field: id
>    [junit4]   2> 752409 T2587 N:127.0.0.1:42328_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 752413 T2587 N:127.0.0.1:42328_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 752428 T2587 N:127.0.0.1:42328_ oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
>    [junit4]   2> 752429 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
>    [junit4]   2> 752430 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/
>    [junit4]   2> 752430 T2587 N:127.0.0.1:42328_ c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b70d03a
>    [junit4]   2> 752432 T2587 N:127.0.0.1:42328_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data
>    [junit4]   2> 752433 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index/
>    [junit4]   2> 752434 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
>    [junit4]   2> 752435 T2587 N:127.0.0.1:42328_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index
>    [junit4]   2> 752435 T2587 N:127.0.0.1:42328_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4225057358391613]
>    [junit4]   2> 752465 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>                commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
>    [junit4]   2> 752466 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 752472 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>    [junit4]   2> 752472 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>    [junit4]   2> 752473 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>    [junit4]   2> 752473 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 752473 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 752474 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>    [junit4]   2> 752474 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 752475 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 752475 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 752497 T2587 N:127.0.0.1:42328_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 752500 T2587 N:127.0.0.1:42328_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 752502 T2587 N:127.0.0.1:42328_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 752504 T2587 N:127.0.0.1:42328_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 752510 T2587 N:127.0.0.1:42328_ c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/get,/admin/properties,/update,/update/json,/config,standard,/update/json/docs,/admin/plugins,/schema,/replication,/admin/segments,/update/csv,/admin/file,/admin/system,/admin/luke,/admin/threads,/admin/logging,/admin/ping
>    [junit4]   2> 752512 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
>    [junit4]   2> 752513 T2587 N:127.0.0.1:42328_ c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
>    [junit4]   2> 752513 T2587 N:127.0.0.1:42328_ c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
>    [junit4]   2> 752514 T2587 N:127.0.0.1:42328_ c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>    [junit4]   2> 752515 T2587 N:127.0.0.1:42328_ c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>    [junit4]   2> 752516 T2587 N:127.0.0.1:42328_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=72.4248046875, floorSegmentMB=1.158203125, forceMergeDeletesPctAllowed=9.55234882367925, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5820428566814219
>    [junit4]   2> 752517 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>                commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
>    [junit4]   2> 752517 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 752518 T2587 N:127.0.0.1:42328_ c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@410b457b[collection1] main
>    [junit4]   2> 752518 T2587 N:127.0.0.1:42328_ c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
>    [junit4]   2> 752519 T2587 N:127.0.0.1:42328_ c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 752520 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
>    [junit4]   2> 752520 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 752520 T2587 N:127.0.0.1:42328_ c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
>    [junit4]   2> 752521 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 752521 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 752521 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 752522 T2587 N:127.0.0.1:42328_ c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
>    [junit4]   2> 752522 T2587 N:127.0.0.1:42328_ c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
>    [junit4]   2> 752523 T2587 N:127.0.0.1:42328_ c:collection1 oasc.CoreContainer.registerCore registering core: collection1
>    [junit4]   2> 752523 T2588 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@410b457b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
>    [junit4]   2> 752524 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42328 collection:control_collection shard:shard1
>    [junit4]   2> 752524 T2557 N:127.0.0.1:42328_ oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
>    [junit4]   2> 752524 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
>    [junit4]   2> 752524 T2557 N:127.0.0.1:42328_ oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>    [junit4]   2> 752527 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 752527 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
>    [junit4]   2> 752528 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 752529 T2594 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@af4db2 name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 752529 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 752529 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
>    [junit4]   2> 752530 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>    [junit4]   2> 752530 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
>    [junit4]   2> 752529 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 752530 T2557 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
>    [junit4]   2> ASYNC  NEW_CORE C271 name=collection1 org.apache.solr.core.SolrCore@5ad7f954 url=http://127.0.0.1:42328/collection1 node=127.0.0.1:42328_ C271_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:42328_, base_url=http://127.0.0.1:42328}
>    [junit4]   2> 752530 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 C271 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42328/collection1/
>    [junit4]   2> 752532 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection"} current state version: 1
>    [junit4]   2> 752532 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 C271 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
>    [junit4]   2> 752533 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 C271 oasc.SyncStrategy.syncToMe http://127.0.0.1:42328/collection1/ has no replicas
>    [junit4]   2> 752533 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42328/collection1/ shard1
>    [junit4]   2> 752533 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
>    [junit4]   2> 752534 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
>    [junit4]   2> 752534 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 752535 T2557 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
>    [junit4]   2> 752537 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 752538 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42328",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active"} current state version: 2
>    [junit4]   2> 752640 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 752640 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 752690 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.register We are http://127.0.0.1:42328/collection1/ and leader is http://127.0.0.1:42328/collection1/
>    [junit4]   2> 752691 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42328
>    [junit4]   2> 752691 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
>    [junit4]   2> 752691 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
>    [junit4]   2> 752691 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 752692 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 752693 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42328_",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42328"} current state version: 3
>    [junit4]   2> 752693 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42328_",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42328"}
>    [junit4]   2> 752796 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 752796 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 753861 T2557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1
>    [junit4]   2> 753864 T2557 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001
>    [junit4]   2> 753864 T2557 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 753867 T2557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42336
>    [junit4]   2> 753867 T2557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=42336, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
>    [junit4]   2> 753868 T2557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
>    [junit4]   2> 753868 T2557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/'
>    [junit4]   2> 753900 T2557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/solr.xml
>    [junit4]   2> 753940 T2557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores
>    [junit4]   2> 753940 T2557 oasc.CoreContainer.<init> New CoreContainer 779197687
>    [junit4]   2> 753940 T2557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/]
>    [junit4]   2> 753941 T2557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/lib
>    [junit4]   2> 753942 T2557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/lib).
>    [junit4]   2> 753951 T2557 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
>    [junit4]   2> 753956 T2557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
>    [junit4]   2> 753957 T2557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 753957 T2557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 753957 T2557 oasc.CoreContainer.load Node Name: 127.0.0.1
>    [junit4]   2> 753957 T2557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42322/solr
>    [junit4]   2> 753958 T2557 oasc.ZkController.checkChrootPath zkHost includes chroot
>    [junit4]   2> 753958 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 753959 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 753960 T2606 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26275435 name:ZooKeeperConnection Watcher:127.0.0.1:42322 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 753961 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 753962 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
>    [junit4]   2> 753964 T2557 N:127.0.0.1:42336_ oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 753965 T2609 N:127.0.0.1:42336_ oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37c8f277 name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 753965 T2557 N:127.0.0.1:42336_ oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 753969 T2557 N:127.0.0.1:42336_ oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
>    [junit4]   2> 754972 T2557 N:127.0.0.1:42336_ oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42336_
>    [junit4]   2> 754973 T2557 N:127.0.0.1:42336_ oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42336_
>    [junit4]   2> 754976 T2557 N:127.0.0.1:42336_ oasc.Overseer.close Overseer (id=null) closing
>    [junit4]   2> 754978 T2557 N:127.0.0.1:42336_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores
>    [junit4]   2> 754981 T2557 N:127.0.0.1:42336_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/, name=collection1, shard=, collection=collection1, loadOnStartup=true, schema=schema.xml, dataDir=data/, config=solrconfig.xml}
>    [junit4]   2> 754981 T2557 N:127.0.0.1:42336_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/
>    [junit4]   2> 754981 T2557 N:127.0.0.1:42336_ oasc.CorePropertiesLocator.discover Found 1 core definitions
>    [junit4]   2> 754982 T2610 N:127.0.0.1:42336_ C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
>    [junit4]   2> 754982 T2610 N:127.0.0.1:42336_ C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 754983 T2610 N:127.0.0.1:42336_ oasc.ZkController.waitForCoreNodeName look for our core node name
>    [junit4]   2> 754983 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 754984 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42336_",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42336"} current state version: 4
>    [junit4]   2> 754984 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42336_",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42336"}
>    [junit4]   2> 754985 T2584 N:127.0.0.1:42328_ oasco.ClusterStateMutator.createCollection building a new cName: collection1
>    [junit4]   2> 754985 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
>    [junit4]   2> 754986 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 754986 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 754986 T2609 N:127.0.0.1:42336_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 755983 T2610 N:127.0.0.1:42336_ oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
>    [junit4]   2> 755984 T2610 N:127.0.0.1:42336_ oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
>    [junit4]   2> 755984 T2610 N:127.0.0.1:42336_ oasc.ZkController.createCollectionZkNode Collection zkNode exists
>    [junit4]   2> 755985 T2610 N:127.0.0.1:42336_ oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
>    [junit4]   2> 755985 T2610 N:127.0.0.1:42336_ oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 755985 T2610 N:127.0.0.1:42336_ oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/'
>    [junit4]   2> 755999 T2610 N:127.0.0.1:42336_ oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
>    [junit4]   2> 756001 T2610 N:127.0.0.1:42336_ oasc.Config.<init> loaded config solrconfig.xml with version 0
>    [junit4]   2> 756010 T2610 N:127.0.0.1:42336_ oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
>    [junit4]   2> 756027 T2610 N:127.0.0.1:42336_ oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
>    [junit4]   2> 756065 T2610 N:127.0.0.1:42336_ oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>    [junit4]   2> 756167 T2610 N:127.0.0.1:42336_ oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
>    [junit4]   2> 756174 T2610 N:127.0.0.1:42336_ oass.IndexSchema.readSchema [collection1] Schema name=test
>    [junit4]   2> 756447 T2610 N:127.0.0.1:42336_ oass.IndexSchema.readSchema default search field in schema is text
>    [junit4]   2> 756450 T2610 N:127.0.0.1:42336_ oass.IndexSchema.readSchema unique key field: id
>    [junit4]   2> 756451 T2610 N:127.0.0.1:42336_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 756455 T2610 N:127.0.0.1:42336_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>    [junit4]   2> 756468 T2610 N:127.0.0.1:42336_ oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
>    [junit4]   2> 756468 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
>    [junit4]   2> 756469 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/
>    [junit4]   2> 756470 T2610 N:127.0.0.1:42336_ c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b70d03a
>    [junit4]   2> 756471 T2610 N:127.0.0.1:42336_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data
>    [junit4]   2> 756473 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index/
>    [junit4]   2> 756474 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
>    [junit4]   2> 756475 T2610 N:127.0.0.1:42336_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index
>    [junit4]   2> 756475 T2610 N:127.0.0.1:42336_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4225057358391613]
>    [junit4]   2> 756480 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>                commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
>    [junit4]   2> 756480 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 756486 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>    [junit4]   2> 756486 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>    [junit4]   2> 756486 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>    [junit4]   2> 756487 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 756487 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 756487 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>    [junit4]   2> 756488 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 756488 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 756488 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 756511 T2610 N:127.0.0.1:42336_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 756513 T2610 N:127.0.0.1:42336_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 756515 T2610 N:127.0.0.1:42336_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 756517 T2610 N:127.0.0.1:42336_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 756525 T2610 N:127.0.0.1:42336_ c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/get,/admin/properties,/update,/update/json,/config,standard,/update/json/docs,/admin/plugins,/schema,/replication,/admin/segments,/update/csv,/admin/file,/admin/system,/admin/luke,/admin/threads,/admin/logging,/admin/ping
>    [junit4]   2> 756527 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
>    [junit4]   2> 756528 T2610 N:127.0.0.1:42336_ c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
>    [junit4]   2> 756528 T2610 N:127.0.0.1:42336_ c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
>    [junit4]   2> 756529 T2610 N:127.0.0.1:42336_ c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>    [junit4]   2> 756530 T2610 N:127.0.0.1:42336_ c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>    [junit4]   2> 756531 T2610 N:127.0.0.1:42336_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=72.4248046875, floorSegmentMB=1.158203125, forceMergeDeletesPctAllowed=9.55234882367925, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5820428566814219
>    [junit4]   2> 756532 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>                commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
>    [junit4]   2> 756532 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 756532 T2610 N:127.0.0.1:42336_ c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5d7f1257[collection1] main
>    [junit4]   2> 756533 T2610 N:127.0.0.1:42336_ c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
>    [junit4]   2> 756533 T2610 N:127.0.0.1:42336_ c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 756534 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
>    [junit4]   2> 756534 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 756534 T2610 N:127.0.0.1:42336_ c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
>    [junit4]   2> 756534 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 756535 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 756535 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 756535 T2610 N:127.0.0.1:42336_ c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
>    [junit4]   2> 756536 T2610 N:127.0.0.1:42336_ c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
>    [junit4]   2> 756536 T2611 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5d7f1257[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
>    [junit4]   2> 756537 T2610 N:127.0.0.1:42336_ c:collection1 oasc.CoreContainer.registerCore registering core: collection1
>    [junit4]   2> 756538 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42336 collection:collection1 shard:shard1
>    [junit4]   2> 756538 T2557 N:127.0.0.1:42336_ oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
>    [junit4]   2> 756538 T2557 N:127.0.0.1:42336_ oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>    [junit4]   2> 756538 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
>    [junit4]   2> 756542 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
>    [junit4]   2> 756543 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 756543 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
>    [junit4]   2> 756543 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>    [junit4]   2> ASYNC  NEW_CORE C272 name=collection1 org.apache.solr.core.SolrCore@46ff4791 url=http://127.0.0.1:42336/collection1 node=127.0.0.1:42336_ C272_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:42336_, base_url=http://127.0.0.1:42336}
>    [junit4]   2> 756544 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 C272 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42336/collection1/
>    [junit4]   2> 756544 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"collection1"} current state version: 5
>    [junit4]   2> 756544 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 C272 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
>    [junit4]   2> 756544 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 C272 oasc.SyncStrategy.syncToMe http://127.0.0.1:42336/collection1/ has no replicas
>    [junit4]   2> 756545 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42336/collection1/ shard1
>    [junit4]   2> 756545 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
>    [junit4]   2> 756545 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 756545 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 756545 T2609 N:127.0.0.1:42336_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 756548 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 756548 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42336",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active"} current state version: 6
>    [junit4]   2> 756652 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 756652 T2609 N:127.0.0.1:42336_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 756652 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 756701 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.register We are http://127.0.0.1:42336/collection1/ and leader is http://127.0.0.1:42336/collection1/
>    [junit4]   2> 756702 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42336
>    [junit4]   2> 756702 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
>    [junit4]   2> 756702 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
>    [junit4]   2> 756702 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
>    [junit4]   2> 756703 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 756704 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42336_",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42336"} current state version: 7
>    [junit4]   2> 756705 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:42336_",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:42336"}
>    [junit4]   2> 756807 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 756807 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 756807 T2609 N:127.0.0.1:42336_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 757893 T2557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1
>    [junit4]   2> 757896 T2557 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001
>    [junit4]   2> 757897 T2557 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 757899 T2557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42343
>    [junit4]   2> 757900 T2557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores, hostPort=42343, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
>    [junit4]   2> 757901 T2557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
>    [junit4]   2> 757901 T2557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/'
>    [junit4]   2> 757933 T2557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/solr.xml
>    [junit4]   2> 757971 T2557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores
>    [junit4]   2> 757972 T2557 oasc.CoreContainer.<init> New CoreContainer 1873691317
>    [junit4]   2> 757972 T2557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/]
>    [junit4]   2> 757973 T2557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/lib
>    [junit4]   2> 757973 T2557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/lib).
>    [junit4]   2> 757983 T2557 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
>    [junit4]   2> 757988 T2557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
>    [junit4]   2> 757988 T2557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 757989 T2557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 757989 T2557 oasc.CoreContainer.load Node Name: 127.0.0.1
>    [junit4]   2> 757989 T2557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42322/solr
>    [junit4]   2> 757989 T2557 oasc.ZkController.checkChrootPath zkHost includes chroot
>    [junit4]   2> 757990 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 757990 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 757992 T2626 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78e0a6f8 name:ZooKeeperConnection Watcher:127.0.0.1:42322 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 757993 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 757994 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
>    [junit4]   2> 757995 T2557 N:127.0.0.1:42343_ oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>    [junit4]   2> 757997 T2629 N:127.0.0.1:42343_ oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20187971 name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 757997 T2557 N:127.0.0.1:42343_ oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>    [junit4]   2> 758001 T2557 N:127.0.0.1:42343_ oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
>    [junit4]   2> 759004 T2557 N:127.0.0.1:42343_ oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42343_
>    [junit4]   2> 759006 T2557 N:127.0.0.1:42343_ oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42343_
>    [junit4]   2> 759009 T2557 N:127.0.0.1:42343_ oasc.Overseer.close Overseer (id=null) closing
>    [junit4]   2> 759012 T2557 N:127.0.0.1:42343_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores
>    [junit4]   2> 759014 T2557 N:127.0.0.1:42343_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1, collection=collection1, schema=schema.xml, dataDir=data/, name=collection1, shard=, loadOnStartup=true, coreNodeName=, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/}
>    [junit4]   2> 759014 T2557 N:127.0.0.1:42343_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/
>    [junit4]   2> 759015 T2557 N:127.0.0.1:42343_ oasc.CorePropertiesLocator.discover Found 1 core definitions
>    [junit4]   2> 759016 T2630 N:127.0.0.1:42343_ C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
>    [junit4]   2> 759016 T2630 N:127.0.0.1:42343_ C:collection1 c:collection1 oasc.ZkController.publish numShards not
>
> [...truncated too long message...]
>
> -001/cores/collection1/data/index.20150418083612427;done=false>>]
>    [junit4]   2> 1467506 T4257 N:127.0.0.1:63274_ecb%2Fay oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/data/index.20150418083612427
>    [junit4]   2> 1467506 T4257 N:127.0.0.1:63274_ecb%2Fay oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/data;done=false>>]
>    [junit4]   2> 1467506 T4257 N:127.0.0.1:63274_ecb%2Fay oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/data
>    [junit4]   2> 1467508 T4134 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/ecb/ay,null}
>    [junit4]   2> 1467667 T4134 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63252 63252
>    [junit4]   2> 1468701 T4135 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63252 63252
>    [junit4]   2> 1468703 T4135 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations:
>    [junit4]   2>        Maximum concurrent create/delete watches above limit:
>    [junit4]   2>
>    [junit4]   2>                4       /solr/clusterstate.json
>    [junit4]   2>                4       /solr/aliases.json
>    [junit4]   2>                3       /solr/configs/conf1
>    [junit4]   2>
>    [junit4]   2>        Maximum concurrent children watches above limit:
>    [junit4]   2>
>    [junit4]   2>                4       /solr/live_nodes
>    [junit4]   2>                3       /solr/overseer/collection-queue-work
>    [junit4]   2>                3       /solr/overseer/queue
>    [junit4]   2>
>    [junit4]   2> 1468703 T4134 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:63266/ecb/ay, target: http://127.0.0.1:63267/ecb/ay
>    [junit4]   2> 1468704 T4134 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:63274/ecb/ay, target: http://127.0.0.1:63275/ecb/ay
>    [junit4]   2> 1468704 T4134 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63258/ecb/ay, target: http://127.0.0.1:63259/ecb/ay
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=74BED4029EC96EE2 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_KW -Dtests.timezone=America/Grand_Turk -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>    [junit4] FAILURE  112s J1 | RecoveryAfterSoftCommitTest.test <<<
>    [junit4]    > Throwable #1: java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
>    [junit4]    >   "control_collection":{
>    [junit4]    >     "autoCreated":"true",
>    [junit4]    >     "replicationFactor":"1",
>    [junit4]    >     "autoAddReplicas":"false",
>    [junit4]    >     "router":{"name":"compositeId"},
>    [junit4]    >     "maxShardsPerNode":"1",
>    [junit4]    >     "shards":{"shard1":{
>    [junit4]    >         "range":"80000000-7fffffff",
>    [junit4]    >         "state":"active",
>    [junit4]    >         "replicas":{"core_node1":{
>    [junit4]    >             "core":"collection1",
>    [junit4]    >             "state":"active",
>    [junit4]    >             "node_name":"127.0.0.1:63258_ecb%2Fay",
>    [junit4]    >             "base_url":"http://127.0.0.1:63258/ecb/ay",
>    [junit4]    >             "leader":"true"}}}}},
>    [junit4]    >   "collection1":{
>    [junit4]    >     "autoCreated":"true",
>    [junit4]    >     "replicationFactor":"1",
>    [junit4]    >     "autoAddReplicas":"false",
>    [junit4]    >     "router":{"name":"compositeId"},
>    [junit4]    >     "maxShardsPerNode":"1",
>    [junit4]    >     "shards":{"shard1":{
>    [junit4]    >         "range":"80000000-7fffffff",
>    [junit4]    >         "state":"active",
>    [junit4]    >         "replicas":{
>    [junit4]    >           "core_node1":{
>    [junit4]    >             "core":"collection1",
>    [junit4]    >             "state":"active",
>    [junit4]    >             "node_name":"127.0.0.1:63266_ecb%2Fay",
>    [junit4]    >             "base_url":"http://127.0.0.1:63266/ecb/ay",
>    [junit4]    >             "leader":"true"},
>    [junit4]    >           "core_node2":{
>    [junit4]    >             "core":"collection1",
>    [junit4]    >             "state":"recovering",
>    [junit4]    >             "node_name":"127.0.0.1:63274_ecb%2Fay",
>    [junit4]    >             "base_url":"http://127.0.0.1:63274/ecb/ay"}}}}}}
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([74BED4029EC96EE2:FCEAEBD83035031A]:0)
>    [junit4]    >        at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
>    [junit4]    >        at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
>    [junit4]    >        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>    [junit4]    >        at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> 1468716 T4134 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=FSTOrd50, timestamp=FSTOrd50, id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=PostingsFormat(name=MockRandom), range_facet_l=FSTOrd50, a_t=FSTOrd50, text=Lucene50(blocksize=128), _version_=PostingsFormat(name=MockRandom)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=ar_KW, timezone=America/Grand_Turk
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=269594032,total=332398592
>    [junit4]   2> NOTE: All tests run in this JVM: [TestElisionMultitermQuery, CollectionsAPIDistributedZkTest, MergeStrategyTest, TestFaceting, VMParamsZkACLAndCredentialsProvidersTest, ChaosMonkeySafeLeaderTest, ZkSolrClientTest, CacheHeaderTest, TestLRUCache, TestReRankQParserPlugin, FileUtilsTest, TestDFRSimilarityFactory, BasicFunctionalityTest, HdfsSyncSliceTest, TestWordDelimiterFilterFactory, TestNamedUpdateProcessors, LoggingHandlerTest, DateMathParserTest, TestMergePolicyConfig, TestIndexingPerformance, SpatialHeatmapFacetsTest, SchemaVersionSpecificBehaviorTest, BinaryUpdateRequestHandlerTest, HdfsRecoveryZkTest, HdfsBasicDistributedZk2Test, SyncSliceTest, ZkCLITest, TestFastOutputStream, MultiTermTest, MoreLikeThisHandlerTest, HdfsWriteToMultipleCollectionsTest, DocExpirationUpdateProcessorFactoryTest, SuggesterTest, QueryEqualityTest, TestShardHandlerFactory, AutoCommitTest, ChaosMonkeyNothingIsSafeTest, TestJmxMonitoredMap, ConcurrentDeleteAndCreateCollectionTest, TestExtendedDismaxParser, TestReloadAndDeleteDocs, SynonymTokenizerTest, TestConfigSets, SolrIndexSplitterTest, TestCollectionAPI, DistributedMLTComponentTest, BlockCacheTest, ExitableDirectoryReaderTest, TestExactStatsCache, SpellCheckComponentTest, TestCollationField, ShardRoutingTest, DOMUtilTest, TestWriterPerf, TestSchemaResource, TestOverriddenPrefixQueryForCustomFieldType, ScriptEngineTest, RankQueryTest, TestMaxScoreQueryParser, RecoveryAfterSoftCommitTest]
>    [junit4] Completed [251/483] on J1 in 111.91s, 1 test, 1 failure <<< FAILURES!
>
> [...truncated 720 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:484: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:229: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:511: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1434: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:991: There were test failures: 483 suites, 1917 tests, 2 failures, 64 ignored (23 assumptions)
>
> Total time: 52 minutes 47 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2966
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 207404135 bytes
> Compression is 0.0%
> Took 39 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2969 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2969/

2 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (25 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (25 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([74BED4029EC96EE2:FCEAEBD83035031A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {   "control_collection":{     "autoCreated":"true",     "replicationFactor":"1",     "autoAddReplicas":"false",     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"collection1",             "state":"active",             "node_name":"127.0.0.1:63258_ecb%2Fay",             "base_url":"http://127.0.0.1:63258/ecb/ay",             "leader":"true"}}}}},   "collection1":{     "autoCreated":"true",     "replicationFactor":"1",     "autoAddReplicas":"false",     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1",             "state":"active",             "node_name":"127.0.0.1:63266_ecb%2Fay",             "base_url":"http://127.0.0.1:63266/ecb/ay",             "leader":"true"},           "core_node2":{             "core":"collection1",             "state":"recovering",             "node_name":"127.0.0.1:63274_ecb%2Fay",             "base_url":"http://127.0.0.1:63274/ecb/ay"}}}}}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
  "control_collection":{
    "autoCreated":"true",
    "replicationFactor":"1",
    "autoAddReplicas":"false",
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "core":"collection1",
            "state":"active",
            "node_name":"127.0.0.1:63258_ecb%2Fay",
            "base_url":"http://127.0.0.1:63258/ecb/ay",
            "leader":"true"}}}}},
  "collection1":{
    "autoCreated":"true",
    "replicationFactor":"1",
    "autoAddReplicas":"false",
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collection1",
            "state":"active",
            "node_name":"127.0.0.1:63266_ecb%2Fay",
            "base_url":"http://127.0.0.1:63266/ecb/ay",
            "leader":"true"},
          "core_node2":{
            "core":"collection1",
            "state":"recovering",
            "node_name":"127.0.0.1:63274_ecb%2Fay",
            "base_url":"http://127.0.0.1:63274/ecb/ay"}}}}}}
	at __randomizedtesting.SeedInfo.seed([74BED4029EC96EE2:FCEAEBD83035031A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9982 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/init-core-data-001
   [junit4]   2> 749637 T2557 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 749643 T2557 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 749643 T2558 oasc.ZkTestServer$2$1.setClientPort client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 749643 T2558 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 749744 T2557 oasc.ZkTestServer.run start zk server on port:42322
   [junit4]   2> 749745 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 749746 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 749749 T2565 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@653864d2 name:ZooKeeperConnection Watcher:127.0.0.1:42322 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 749750 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 749750 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 749751 T2557 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 749754 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 749755 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 749756 T2568 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c2e72e name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 749757 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 749757 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 749758 T2557 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 749760 T2557 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 749761 T2557 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 749762 T2557 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 749763 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 749764 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 749766 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 749767 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 749869 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 749870 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 749871 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 749872 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 749873 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 749873 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 749874 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 749875 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 749876 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 749876 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 749877 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 749878 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 749879 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 749879 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 749880 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 749881 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 749882 T2557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 749882 T2557 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 750801 T2557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1
   [junit4]   2> 750804 T2557 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 750809 T2557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42328
   [junit4]   2> 750809 T2557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=42328, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/tempDir-001/control/data, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores}
   [junit4]   2> 750809 T2557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 750810 T2557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/'
   [junit4]   2> 750839 T2557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/solr.xml
   [junit4]   2> 750874 T2557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores
   [junit4]   2> 750875 T2557 oasc.CoreContainer.<init> New CoreContainer 616784085
   [junit4]   2> 750875 T2557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/]
   [junit4]   2> 750875 T2557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/lib
   [junit4]   2> 750876 T2557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/lib).
   [junit4]   2> 750885 T2557 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 750889 T2557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 750890 T2557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 750890 T2557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 750890 T2557 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 750891 T2557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42322/solr
   [junit4]   2> 750891 T2557 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 750891 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 750892 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 750893 T2580 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d64d521 name:ZooKeeperConnection Watcher:127.0.0.1:42322 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 750894 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 750895 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 750897 T2557 N:127.0.0.1:42328_ oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 750898 T2583 N:127.0.0.1:42328_ oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5449a446 name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 750898 T2557 N:127.0.0.1:42328_ oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 750899 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 750901 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 750902 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 750904 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 750905 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 750907 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 750908 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 750909 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 750910 T2557 N:127.0.0.1:42328_ oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42328_
   [junit4]   2> 750910 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42328_
   [junit4]   2> 750911 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 750912 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 750913 T2557 N:127.0.0.1:42328_ oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 750914 T2557 N:127.0.0.1:42328_ oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:42328_
   [junit4]   2> 750915 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 750916 T2557 N:127.0.0.1:42328_ oasc.Overseer.start Overseer (id=93674533455265795-127.0.0.1:42328_-n_0000000000) starting
   [junit4]   2> 750917 T2557 N:127.0.0.1:42328_ oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 750921 T2557 N:127.0.0.1:42328_ oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 750922 T2585 N:127.0.0.1:42328_ oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 750922 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 750922 T2557 N:127.0.0.1:42328_ oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 750925 T2557 N:127.0.0.1:42328_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores
   [junit4]   2> 750928 T2557 N:127.0.0.1:42328_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, schema=schema.xml, coreNodeName=, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/, collection=control_collection, dataDir=data/, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1, loadOnStartup=true, config=solrconfig.xml}
   [junit4]   2> 750928 T2557 N:127.0.0.1:42328_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/
   [junit4]   2> 750929 T2557 N:127.0.0.1:42328_ oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 750929 T2587 N:127.0.0.1:42328_ C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 750929 T2587 N:127.0.0.1:42328_ C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 750930 T2587 N:127.0.0.1:42328_ oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 750930 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 750931 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42328_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42328"} current state version: 0
   [junit4]   2> 750931 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42328_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42328"}
   [junit4]   2> 750931 T2584 N:127.0.0.1:42328_ oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 750932 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 750933 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 751930 T2587 N:127.0.0.1:42328_ oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 751931 T2587 N:127.0.0.1:42328_ oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 751931 T2587 N:127.0.0.1:42328_ oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 751932 T2587 N:127.0.0.1:42328_ oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 751932 T2587 N:127.0.0.1:42328_ oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 751932 T2587 N:127.0.0.1:42328_ oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/'
   [junit4]   2> 751947 T2587 N:127.0.0.1:42328_ oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 751949 T2587 N:127.0.0.1:42328_ oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 751957 T2587 N:127.0.0.1:42328_ oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 751974 T2587 N:127.0.0.1:42328_ oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 752022 T2587 N:127.0.0.1:42328_ oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 752124 T2587 N:127.0.0.1:42328_ oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 752132 T2587 N:127.0.0.1:42328_ oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 752405 T2587 N:127.0.0.1:42328_ oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 752407 T2587 N:127.0.0.1:42328_ oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 752409 T2587 N:127.0.0.1:42328_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 752413 T2587 N:127.0.0.1:42328_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 752428 T2587 N:127.0.0.1:42328_ oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 752429 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 752430 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/
   [junit4]   2> 752430 T2587 N:127.0.0.1:42328_ c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b70d03a
   [junit4]   2> 752432 T2587 N:127.0.0.1:42328_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data
   [junit4]   2> 752433 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index/
   [junit4]   2> 752434 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 752435 T2587 N:127.0.0.1:42328_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index
   [junit4]   2> 752435 T2587 N:127.0.0.1:42328_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4225057358391613]
   [junit4]   2> 752465 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 752466 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 752472 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 752472 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 752473 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 752473 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 752473 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 752474 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 752474 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 752475 T2587 N:127.0.0.1:42328_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 752475 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 752497 T2587 N:127.0.0.1:42328_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 752500 T2587 N:127.0.0.1:42328_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 752502 T2587 N:127.0.0.1:42328_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 752504 T2587 N:127.0.0.1:42328_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 752510 T2587 N:127.0.0.1:42328_ c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/get,/admin/properties,/update,/update/json,/config,standard,/update/json/docs,/admin/plugins,/schema,/replication,/admin/segments,/update/csv,/admin/file,/admin/system,/admin/luke,/admin/threads,/admin/logging,/admin/ping
   [junit4]   2> 752512 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 752513 T2587 N:127.0.0.1:42328_ c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 752513 T2587 N:127.0.0.1:42328_ c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 752514 T2587 N:127.0.0.1:42328_ c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 752515 T2587 N:127.0.0.1:42328_ c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 752516 T2587 N:127.0.0.1:42328_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=72.4248046875, floorSegmentMB=1.158203125, forceMergeDeletesPctAllowed=9.55234882367925, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5820428566814219
   [junit4]   2> 752517 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 752517 T2587 N:127.0.0.1:42328_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 752518 T2587 N:127.0.0.1:42328_ c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@410b457b[collection1] main
   [junit4]   2> 752518 T2587 N:127.0.0.1:42328_ c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 752519 T2587 N:127.0.0.1:42328_ c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 752520 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 752520 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 752520 T2587 N:127.0.0.1:42328_ c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 752521 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 752521 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 752521 T2587 N:127.0.0.1:42328_ c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 752522 T2587 N:127.0.0.1:42328_ c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 752522 T2587 N:127.0.0.1:42328_ c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 752523 T2587 N:127.0.0.1:42328_ c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 752523 T2588 N:127.0.0.1:42328_ c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@410b457b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 752524 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42328 collection:control_collection shard:shard1
   [junit4]   2> 752524 T2557 N:127.0.0.1:42328_ oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 752524 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 752524 T2557 N:127.0.0.1:42328_ oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 752527 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 752527 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 752528 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 752529 T2594 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@af4db2 name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 752529 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 752529 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 752530 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 752530 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 752529 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 752530 T2557 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C271 name=collection1 org.apache.solr.core.SolrCore@5ad7f954 url=http://127.0.0.1:42328/collection1 node=127.0.0.1:42328_ C271_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:42328_, base_url=http://127.0.0.1:42328}
   [junit4]   2> 752530 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 C271 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42328/collection1/
   [junit4]   2> 752532 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 752532 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 C271 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 752533 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 C271 oasc.SyncStrategy.syncToMe http://127.0.0.1:42328/collection1/ has no replicas
   [junit4]   2> 752533 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42328/collection1/ shard1
   [junit4]   2> 752533 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 752534 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 752534 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 752535 T2557 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 752537 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 752538 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42328",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 752640 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 752640 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 752690 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.register We are http://127.0.0.1:42328/collection1/ and leader is http://127.0.0.1:42328/collection1/
   [junit4]   2> 752691 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42328
   [junit4]   2> 752691 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 752691 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 752691 T2591 N:127.0.0.1:42328_ C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 752692 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 752693 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42328_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42328"} current state version: 3
   [junit4]   2> 752693 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42328_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42328"}
   [junit4]   2> 752796 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 752796 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753861 T2557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1
   [junit4]   2> 753864 T2557 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001
   [junit4]   2> 753864 T2557 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 753867 T2557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42336
   [junit4]   2> 753867 T2557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=42336, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 753868 T2557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 753868 T2557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/'
   [junit4]   2> 753900 T2557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/solr.xml
   [junit4]   2> 753940 T2557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores
   [junit4]   2> 753940 T2557 oasc.CoreContainer.<init> New CoreContainer 779197687
   [junit4]   2> 753940 T2557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/]
   [junit4]   2> 753941 T2557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/lib
   [junit4]   2> 753942 T2557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/lib).
   [junit4]   2> 753951 T2557 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 753956 T2557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 753957 T2557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 753957 T2557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 753957 T2557 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 753957 T2557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42322/solr
   [junit4]   2> 753958 T2557 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 753958 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 753959 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 753960 T2606 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26275435 name:ZooKeeperConnection Watcher:127.0.0.1:42322 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 753961 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 753962 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 753964 T2557 N:127.0.0.1:42336_ oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 753965 T2609 N:127.0.0.1:42336_ oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37c8f277 name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 753965 T2557 N:127.0.0.1:42336_ oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 753969 T2557 N:127.0.0.1:42336_ oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 754972 T2557 N:127.0.0.1:42336_ oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42336_
   [junit4]   2> 754973 T2557 N:127.0.0.1:42336_ oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42336_
   [junit4]   2> 754976 T2557 N:127.0.0.1:42336_ oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 754978 T2557 N:127.0.0.1:42336_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores
   [junit4]   2> 754981 T2557 N:127.0.0.1:42336_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/, name=collection1, shard=, collection=collection1, loadOnStartup=true, schema=schema.xml, dataDir=data/, config=solrconfig.xml}
   [junit4]   2> 754981 T2557 N:127.0.0.1:42336_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/
   [junit4]   2> 754981 T2557 N:127.0.0.1:42336_ oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 754982 T2610 N:127.0.0.1:42336_ C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 754982 T2610 N:127.0.0.1:42336_ C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 754983 T2610 N:127.0.0.1:42336_ oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 754983 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 754984 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42336_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42336"} current state version: 4
   [junit4]   2> 754984 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42336_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42336"}
   [junit4]   2> 754985 T2584 N:127.0.0.1:42328_ oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 754985 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 754986 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 754986 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 754986 T2609 N:127.0.0.1:42336_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 755983 T2610 N:127.0.0.1:42336_ oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 755984 T2610 N:127.0.0.1:42336_ oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 755984 T2610 N:127.0.0.1:42336_ oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 755985 T2610 N:127.0.0.1:42336_ oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 755985 T2610 N:127.0.0.1:42336_ oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 755985 T2610 N:127.0.0.1:42336_ oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/'
   [junit4]   2> 755999 T2610 N:127.0.0.1:42336_ oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 756001 T2610 N:127.0.0.1:42336_ oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 756010 T2610 N:127.0.0.1:42336_ oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 756027 T2610 N:127.0.0.1:42336_ oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 756065 T2610 N:127.0.0.1:42336_ oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 756167 T2610 N:127.0.0.1:42336_ oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 756174 T2610 N:127.0.0.1:42336_ oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 756447 T2610 N:127.0.0.1:42336_ oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 756450 T2610 N:127.0.0.1:42336_ oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 756451 T2610 N:127.0.0.1:42336_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 756455 T2610 N:127.0.0.1:42336_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 756468 T2610 N:127.0.0.1:42336_ oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 756468 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 756469 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 756470 T2610 N:127.0.0.1:42336_ c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b70d03a
   [junit4]   2> 756471 T2610 N:127.0.0.1:42336_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data
   [junit4]   2> 756473 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 756474 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 756475 T2610 N:127.0.0.1:42336_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 756475 T2610 N:127.0.0.1:42336_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4225057358391613]
   [junit4]   2> 756480 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 756480 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 756486 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 756486 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 756486 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 756487 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 756487 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 756487 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 756488 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 756488 T2610 N:127.0.0.1:42336_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 756488 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 756511 T2610 N:127.0.0.1:42336_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 756513 T2610 N:127.0.0.1:42336_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 756515 T2610 N:127.0.0.1:42336_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 756517 T2610 N:127.0.0.1:42336_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 756525 T2610 N:127.0.0.1:42336_ c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/get,/admin/properties,/update,/update/json,/config,standard,/update/json/docs,/admin/plugins,/schema,/replication,/admin/segments,/update/csv,/admin/file,/admin/system,/admin/luke,/admin/threads,/admin/logging,/admin/ping
   [junit4]   2> 756527 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 756528 T2610 N:127.0.0.1:42336_ c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 756528 T2610 N:127.0.0.1:42336_ c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 756529 T2610 N:127.0.0.1:42336_ c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 756530 T2610 N:127.0.0.1:42336_ c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 756531 T2610 N:127.0.0.1:42336_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=72.4248046875, floorSegmentMB=1.158203125, forceMergeDeletesPctAllowed=9.55234882367925, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5820428566814219
   [junit4]   2> 756532 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 756532 T2610 N:127.0.0.1:42336_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 756532 T2610 N:127.0.0.1:42336_ c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5d7f1257[collection1] main
   [junit4]   2> 756533 T2610 N:127.0.0.1:42336_ c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 756533 T2610 N:127.0.0.1:42336_ c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 756534 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 756534 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 756534 T2610 N:127.0.0.1:42336_ c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 756534 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 756535 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 756535 T2610 N:127.0.0.1:42336_ c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 756535 T2610 N:127.0.0.1:42336_ c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 756536 T2610 N:127.0.0.1:42336_ c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 756536 T2611 N:127.0.0.1:42336_ c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5d7f1257[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 756537 T2610 N:127.0.0.1:42336_ c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 756538 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42336 collection:collection1 shard:shard1
   [junit4]   2> 756538 T2557 N:127.0.0.1:42336_ oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 756538 T2557 N:127.0.0.1:42336_ oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 756538 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 756542 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 756543 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 756543 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 756543 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C272 name=collection1 org.apache.solr.core.SolrCore@46ff4791 url=http://127.0.0.1:42336/collection1 node=127.0.0.1:42336_ C272_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:42336_, base_url=http://127.0.0.1:42336}
   [junit4]   2> 756544 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 C272 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42336/collection1/
   [junit4]   2> 756544 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 756544 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 C272 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 756544 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 C272 oasc.SyncStrategy.syncToMe http://127.0.0.1:42336/collection1/ has no replicas
   [junit4]   2> 756545 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42336/collection1/ shard1
   [junit4]   2> 756545 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 756545 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756545 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756545 T2609 N:127.0.0.1:42336_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756548 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 756548 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42336",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 756652 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756652 T2609 N:127.0.0.1:42336_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756652 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756701 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.register We are http://127.0.0.1:42336/collection1/ and leader is http://127.0.0.1:42336/collection1/
   [junit4]   2> 756702 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42336
   [junit4]   2> 756702 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 756702 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 756702 T2614 N:127.0.0.1:42336_ C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 756703 T2583 N:127.0.0.1:42328_ oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 756704 T2584 N:127.0.0.1:42328_ oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42336_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42336"} current state version: 7
   [junit4]   2> 756705 T2584 N:127.0.0.1:42328_ oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42336_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42336"}
   [junit4]   2> 756807 T2583 N:127.0.0.1:42328_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756807 T2594 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756807 T2609 N:127.0.0.1:42336_ oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 757893 T2557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1
   [junit4]   2> 757896 T2557 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001
   [junit4]   2> 757897 T2557 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 757899 T2557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42343
   [junit4]   2> 757900 T2557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores, hostPort=42343, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
   [junit4]   2> 757901 T2557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 757901 T2557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/'
   [junit4]   2> 757933 T2557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/solr.xml
   [junit4]   2> 757971 T2557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores
   [junit4]   2> 757972 T2557 oasc.CoreContainer.<init> New CoreContainer 1873691317
   [junit4]   2> 757972 T2557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/]
   [junit4]   2> 757973 T2557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/lib
   [junit4]   2> 757973 T2557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/lib).
   [junit4]   2> 757983 T2557 oashc.HttpShardHandlerFactory.init created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 757988 T2557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 757988 T2557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 757989 T2557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 757989 T2557 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 757989 T2557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42322/solr
   [junit4]   2> 757989 T2557 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 757990 T2557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 757990 T2557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 757992 T2626 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78e0a6f8 name:ZooKeeperConnection Watcher:127.0.0.1:42322 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 757993 T2557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 757994 T2557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 757995 T2557 N:127.0.0.1:42343_ oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 757997 T2629 N:127.0.0.1:42343_ oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20187971 name:ZooKeeperConnection Watcher:127.0.0.1:42322/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 757997 T2557 N:127.0.0.1:42343_ oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 758001 T2557 N:127.0.0.1:42343_ oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 759004 T2557 N:127.0.0.1:42343_ oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42343_
   [junit4]   2> 759006 T2557 N:127.0.0.1:42343_ oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42343_
   [junit4]   2> 759009 T2557 N:127.0.0.1:42343_ oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 759012 T2557 N:127.0.0.1:42343_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores
   [junit4]   2> 759014 T2557 N:127.0.0.1:42343_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1, collection=collection1, schema=schema.xml, dataDir=data/, name=collection1, shard=, loadOnStartup=true, coreNodeName=, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/}
   [junit4]   2> 759014 T2557 N:127.0.0.1:42343_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/
   [junit4]   2> 759015 T2557 N:127.0.0.1:42343_ oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 759016 T2630 N:127.0.0.1:42343_ C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 759016 T2630 N:127.0.0.1:42343_ C:collection1 c:collection1 oasc.ZkController.publish numShards not

[...truncated too long message...]

-001/cores/collection1/data/index.20150418083612427;done=false>>]
   [junit4]   2> 1467506 T4257 N:127.0.0.1:63274_ecb%2Fay oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/data/index.20150418083612427
   [junit4]   2> 1467506 T4257 N:127.0.0.1:63274_ecb%2Fay oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 1467506 T4257 N:127.0.0.1:63274_ecb%2Fay oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001/shard-2-001/cores/collection1/data
   [junit4]   2> 1467508 T4134 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/ecb/ay,null}
   [junit4]   2> 1467667 T4134 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63252 63252
   [junit4]   2> 1468701 T4135 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63252 63252
   [junit4]   2> 1468703 T4135 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1468703 T4134 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:63266/ecb/ay, target: http://127.0.0.1:63267/ecb/ay
   [junit4]   2> 1468704 T4134 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:63274/ecb/ay, target: http://127.0.0.1:63275/ecb/ay
   [junit4]   2> 1468704 T4134 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63258/ecb/ay, target: http://127.0.0.1:63259/ecb/ay
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=74BED4029EC96EE2 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_KW -Dtests.timezone=America/Grand_Turk -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE  112s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
   [junit4]    >   "control_collection":{
   [junit4]    >     "autoCreated":"true",
   [junit4]    >     "replicationFactor":"1",
   [junit4]    >     "autoAddReplicas":"false",
   [junit4]    >     "router":{"name":"compositeId"},
   [junit4]    >     "maxShardsPerNode":"1",
   [junit4]    >     "shards":{"shard1":{
   [junit4]    >         "range":"80000000-7fffffff",
   [junit4]    >         "state":"active",
   [junit4]    >         "replicas":{"core_node1":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"active",
   [junit4]    >             "node_name":"127.0.0.1:63258_ecb%2Fay",
   [junit4]    >             "base_url":"http://127.0.0.1:63258/ecb/ay",
   [junit4]    >             "leader":"true"}}}}},
   [junit4]    >   "collection1":{
   [junit4]    >     "autoCreated":"true",
   [junit4]    >     "replicationFactor":"1",
   [junit4]    >     "autoAddReplicas":"false",
   [junit4]    >     "router":{"name":"compositeId"},
   [junit4]    >     "maxShardsPerNode":"1",
   [junit4]    >     "shards":{"shard1":{
   [junit4]    >         "range":"80000000-7fffffff",
   [junit4]    >         "state":"active",
   [junit4]    >         "replicas":{
   [junit4]    >           "core_node1":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"active",
   [junit4]    >             "node_name":"127.0.0.1:63266_ecb%2Fay",
   [junit4]    >             "base_url":"http://127.0.0.1:63266/ecb/ay",
   [junit4]    >             "leader":"true"},
   [junit4]    >           "core_node2":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"recovering",
   [junit4]    >             "node_name":"127.0.0.1:63274_ecb%2Fay",
   [junit4]    >             "base_url":"http://127.0.0.1:63274/ecb/ay"}}}}}}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([74BED4029EC96EE2:FCEAEBD83035031A]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1468716 T4134 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 74BED4029EC96EE2-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=FSTOrd50, timestamp=FSTOrd50, id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=PostingsFormat(name=MockRandom), range_facet_l=FSTOrd50, a_t=FSTOrd50, text=Lucene50(blocksize=128), _version_=PostingsFormat(name=MockRandom)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=ar_KW, timezone=America/Grand_Turk
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=269594032,total=332398592
   [junit4]   2> NOTE: All tests run in this JVM: [TestElisionMultitermQuery, CollectionsAPIDistributedZkTest, MergeStrategyTest, TestFaceting, VMParamsZkACLAndCredentialsProvidersTest, ChaosMonkeySafeLeaderTest, ZkSolrClientTest, CacheHeaderTest, TestLRUCache, TestReRankQParserPlugin, FileUtilsTest, TestDFRSimilarityFactory, BasicFunctionalityTest, HdfsSyncSliceTest, TestWordDelimiterFilterFactory, TestNamedUpdateProcessors, LoggingHandlerTest, DateMathParserTest, TestMergePolicyConfig, TestIndexingPerformance, SpatialHeatmapFacetsTest, SchemaVersionSpecificBehaviorTest, BinaryUpdateRequestHandlerTest, HdfsRecoveryZkTest, HdfsBasicDistributedZk2Test, SyncSliceTest, ZkCLITest, TestFastOutputStream, MultiTermTest, MoreLikeThisHandlerTest, HdfsWriteToMultipleCollectionsTest, DocExpirationUpdateProcessorFactoryTest, SuggesterTest, QueryEqualityTest, TestShardHandlerFactory, AutoCommitTest, ChaosMonkeyNothingIsSafeTest, TestJmxMonitoredMap, ConcurrentDeleteAndCreateCollectionTest, TestExtendedDismaxParser, TestReloadAndDeleteDocs, SynonymTokenizerTest, TestConfigSets, SolrIndexSplitterTest, TestCollectionAPI, DistributedMLTComponentTest, BlockCacheTest, ExitableDirectoryReaderTest, TestExactStatsCache, SpellCheckComponentTest, TestCollationField, ShardRoutingTest, DOMUtilTest, TestWriterPerf, TestSchemaResource, TestOverriddenPrefixQueryForCustomFieldType, ScriptEngineTest, RankQueryTest, TestMaxScoreQueryParser, RecoveryAfterSoftCommitTest]
   [junit4] Completed [251/483] on J1 in 111.91s, 1 test, 1 failure <<< FAILURES!

[...truncated 720 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:511: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:991: There were test failures: 483 suites, 1917 tests, 2 failures, 64 ignored (23 assumptions)

Total time: 52 minutes 47 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2966
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 207404135 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure