You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2016/10/12 15:24:44 UTC

[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 174 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/174/

2 tests failed.
FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=24687, name=Thread-11778, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=24687, name=Thread-11778, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:51065/xvo/mi/collection1
	at __randomizedtesting.SeedInfo.seed([C3DFCCDCCB98882]:0)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:644)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:51065/xvo/mi/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:642)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
	... 5 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:35901

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:35901
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:437)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:179)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
	... 47 more




Build Log:
[...truncated 11659 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20161012_135246_796.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/heapdumps/java_pid6257.hprof ...
   [junit4] Heap dump file created [656594589 bytes in 3.728 secs]
   [junit4] <<< JVM J0: EOF ----

   [junit4] JVM J0: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20161012_135246_796.syserr
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] 
   [junit4] Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "OldIndexDirectoryCleanupThreadForCore-awholynewstresscollection_collection1_2_shard3_replica2"
   [junit4] WARN: Unhandled exception in event serialization. -> java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] <<< JVM J0: EOF ----

[...truncated 170 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/init-core-data-001
   [junit4]   2> 1721364 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[C3DFCCDCCB98882]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1721364 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[C3DFCCDCCB98882]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 1721366 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1721366 INFO  (Thread-11245) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1721366 INFO  (Thread-11245) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1721466 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.ZkTestServer start zk server on port:57186
   [junit4]   2> 1721472 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x157b94503d50000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1721477 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1721479 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1721480 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1721481 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1721482 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1721483 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1721484 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1721485 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1721486 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1721487 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1721488 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1721491 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x157b94503d50002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1721556 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/cores/collection1
   [junit4]   2> 1721558 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1721559 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5f063735{/,null,AVAILABLE}
   [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.ServerConnector Started ServerConnector@1d14519b{SSL,[ssl, http/1.1]}{127.0.0.1:33163}
   [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server Started @1723802ms
   [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/tempDir-001/control/data, hostContext=/, hostPort=33163, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/cores}
   [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-12T14:21:30.648Z
   [junit4]   2> 1721564 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1721564 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/solr.xml
   [junit4]   2> 1721577 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1721577 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
   [junit4]   2> 1721592 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:33163_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33163_
   [junit4]   2> 1721593 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:33163_    ] o.a.s.c.Overseer Overseer (id=96749623079927813-127.0.0.1:33163_-n_0000000000) starting
   [junit4]   2> 1721596 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:33163_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33163_
   [junit4]   2> 1721597 INFO  (OverseerStateUpdate-96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1721605 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:33163_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/cores
   [junit4]   2> 1721605 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:33163_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1721606 INFO  (OverseerStateUpdate-96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 1721617 WARN  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1721618 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1721627 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1721692 WARN  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1721693 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1721700 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1721701 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/cores/collection1/data/]
   [junit4]   2> 1721701 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@365bd187
   [junit4]   2> 1721701 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1721704 WARN  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 1721709 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1721709 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1721709 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1721709 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1721710 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625, forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1721710 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@28f83cc2[collection1] main]
   [junit4]   2> 1721710 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1721711 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1721711 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1721711 INFO  (searcherExecutor-5526-thread-1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@28f83cc2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1721711 INFO  (coreLoadExecutor-5525-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1547993969639555072
   [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:33163/collection1/
   [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:33163/collection1/ has no replicas
   [junit4]   2> 1721719 INFO  (coreZkRegister-5518-thread-1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:33163/collection1/ shard1
   [junit4]   2> 1721870 INFO  (coreZkRegister-5518-thread-1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1722109 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1722110 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1722175 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/cores/collection1
   [junit4]   2> 1722176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001
   [junit4]   2> 1722177 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1722178 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@78549ff9{/,null,AVAILABLE}
   [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.ServerConnector Started ServerConnector@78092f07{SSL,[ssl, http/1.1]}{127.0.0.1:35901}
   [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server Started @1724420ms
   [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=35901, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/cores}
   [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-12T14:21:31.266Z
   [junit4]   2> 1722182 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1722182 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/solr.xml
   [junit4]   2> 1722186 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1722187 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
   [junit4]   2> 1722192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:35901_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1722195 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:35901_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35901_
   [junit4]   2> 1722196 INFO  (zkCallback-2718-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1722196 INFO  (zkCallback-2727-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1722196 INFO  (zkCallback-2722-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1722207 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:35901_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/cores
   [junit4]   2> 1722207 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:35901_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1722208 INFO  (OverseerStateUpdate-96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 1723221 WARN  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1723222 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1723232 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1723302 WARN  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1723304 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1723313 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1723313 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 1723313 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@365bd187
   [junit4]   2> 1723314 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1723317 WARN  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 1723322 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1723322 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1723323 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1723323 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1723324 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625, forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1723324 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@61147bc7[collection1] main]
   [junit4]   2> 1723325 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1723325 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1723325 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1723326 INFO  (searcherExecutor-5537-thread-1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@61147bc7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1723326 INFO  (coreLoadExecutor-5536-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1547993971333005312
   [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:35901/collection1/
   [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:35901/collection1/ has no replicas
   [junit4]   2> 1723334 INFO  (coreZkRegister-5531-thread-1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:35901/collection1/ shard2
   [junit4]   2> 1723484 INFO  (coreZkRegister-5531-thread-1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1723777 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/cores/collection1
   [junit4]   2> 1723777 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001
   [junit4]   2> 1723779 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1723780 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@70afaa33{/,null,AVAILABLE}
   [junit4]   2> 1723781 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.ServerConnector Started ServerConnector@518924{SSL,[ssl, http/1.1]}{127.0.0.1:59763}
   [junit4]   2> 1723781 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server Started @1726022ms
   [junit4]   2> 1723781 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=59763, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/cores}
   [junit4]   2> 1723782 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 1723782 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1723782 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1723782 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-12T14:21:32.868Z
   [junit4]   2> 1723786 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1723786 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/solr.xml
   [junit4]   2> 1723791 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1723792 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
   [junit4]   2> 1723800 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:59763_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1723802 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:59763_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59763_
   [junit4]   2> 1723803 INFO  (zkCallback-2722-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1723803 INFO  (zkCallback-2727-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1723803 INFO  (zkCallback-2718-thread-1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1723805 INFO  (zkCallback-2733-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1723810 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:59763_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/cores
   [junit4]   2> 1723810 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:59763_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1723811 INFO  (OverseerStateUpdate-96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 1724819 WARN  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1724820 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1724830 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1724902 WARN  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1724904 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1724913 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1724913 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 1724913 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@365bd187
   [junit4]   2> 1724914 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1724917 WARN  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 1724923 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1724923 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1724923 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1724923 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1724924 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625, forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1724924 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5a83c8ea[collection1] main]
   [junit4]   2> 1724925 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1724925 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1724925 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1724926 INFO  (searcherExecutor-5548-thread-1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5a83c8ea[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1724926 INFO  (coreLoadExecutor-5547-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1547993973010726912
   [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:59763/collection1/
   [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:59763/collection1/ has no replicas
   [junit4]   2> 1724934 INFO  (coreZkRegister-5542-thread-1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:59763/collection1/ shard1
   [junit4]   2> 1725085 INFO  (coreZkRegister-5542-thread-1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1725376 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/cores/collection1
   [junit4]   2> 1725376 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001
   [junit4]   2> 1725377 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1725379 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@60026e9{/,null,AVAILABLE}
   [junit4]   2> 1725381 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.ServerConnector Started ServerConnector@7c6e0e6f{SSL,[ssl, http/1.1]}{127.0.0.1:40922}
   [junit4]   2> 1725381 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server Started @1727622ms
   [junit4]   2> 1725381 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=40922, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/cores}
   [junit4]   2> 1725382 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 1725382 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1725382 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1725382 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-12T14:21:34.468Z
   [junit4]   2> 1725385 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1725385 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/solr.xml
   [junit4]   2> 1725389 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1725390 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
   [junit4]   2> 1725398 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:40922_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1725401 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:40922_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40922_
   [junit4]   2> 1725402 INFO  (zkCallback-2733-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1725402 INFO  (zkCallback-2722-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1725402 INFO  (zkCallback-2739-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1725402 INFO  (zkCallback-2727-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1725402 INFO  (zkCallback-2718-thread-2-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1725414 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:40922_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/cores
   [junit4]   2> 1725414 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:40922_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1725415 INFO  (OverseerStateUpdate-96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 1726422 WARN  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1726423 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1726433 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1726500 WARN  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1726502 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1726510 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1726510 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/cores/collection1/data/]
   [junit4]   2> 1726510 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@365bd187
   [junit4]   2> 1726511 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1726514 WARN  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 1726521 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1726521 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1726521 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1726521 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1726522 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625, forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1726522 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2c3fcd7e[collection1] main]
   [junit4]   2> 1726523 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1726523 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1726523 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1726524 INFO  (searcherExecutor-5559-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2c3fcd7e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1726524 INFO  (coreLoadExecutor-5558-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1547993974686351360
   [junit4]   2> 1726528 INFO  (coreZkRegister-5553-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 1726528 INFO  (updateExecutor-2736-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 1726528 INFO  (recoveryExecutor-2737-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 1726529 INFO  (recoveryExecutor-2737-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 1726529 INFO  (recoveryExecutor-2737-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
   [junit4]   2> 1726529 INFO  (recoveryExecutor-2737-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 1726529 INFO  (recoveryExecutor-2737-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [https://127.0.0.1:35901/collection1/] and I am [https://127.0.0.1:40922/collection1/]
   [junit4]   2> 1726531 INFO  (recoveryExecutor-2737-thread-1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1 r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [https://127.0.0.1:35901]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:40922_&coreNodeName=core_node3&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 1726588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 1726588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_    ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard2 of collection1) have state: recovering
   [junit4]   2> 1726588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:40922_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"https://127.0.0.1:40922","node_name":"127.0.0.1:40922_","state":"down"}
   [junit4]   2> 1726979 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/cores/collection1
   [junit4]   2> 1726980 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001
   [junit4]   2> 1726981 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1726983 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2d232e59{/,null,AVAILABLE}
   [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.ServerConnector Started ServerConnector@248524fa{SSL,[ssl, http/1.1]}{127.0.0.1:40836}
   [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.e.j.s.Server Started @1729225ms
   [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=40836, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/cores}
   [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-12T14:21:36.070Z
   [junit4]   2> 1726987 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 1726987 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/solr.xml
   [junit4]   2> 1726994 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1726995 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
   [junit4]   2> 1727001 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:40836_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1727004 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:40836_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40836_
   [junit4]   2> 1727005 INFO  (zkCallback-2727-thread-1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1727005 INFO  (zkCallback-2733-thread-1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1727005 INFO  (zkCallback-2718-thread-3-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1727005 INFO  (zkCallback-2739-thread-1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1727005 INFO  (zkCallback-2746-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1727005 INFO  (zkCallback-2722-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1727014 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:40836_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/cores
   [junit4]   2> 1727014 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882]) [n:127.0.0.1:40836_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 1727016 INFO  (OverseerStateUpdate-96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 1727588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:40922_, coreNodeName=core_node3, onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"collection1","base_url":"https://127.0.0.1:40922","node_name":"127.0.0.1:40922_","state":"recovering"}
   [junit4]   2> 1727588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 1727589 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:40922_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node3&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1000
   [junit4]   2> 1728024 WARN  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1728025 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 1728047 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1728107 WARN  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1728109 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1728116 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1728117 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/cores/collection1/data/]
   [junit4]   2> 1728117 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@365bd187
   [junit4]   2> 1728117 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1728121 WARN  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 1728127 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1728127 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1728135 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1728135 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1728136 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625, forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1728136 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@40d4464b[collection1] main]
   [junit4]   2> 1728137 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1728137 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1728137 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1728137 INFO  (searcherExecutor-5570-thread-1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@40d4464b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1728138 INFO  (coreLoadExecutor-5569-thread-1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1547993976378753024
   [junit4]   2> 1728141 INFO  (coreZkRegister-5564-thread-1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 1728142 INFO  (updateExecutor-2743-thread-1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 1728142 INFO  (recoveryExecutor-2744-thread-1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 1728142 INFO  (recoveryExecutor-2744-thread-1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 1728142 INFO  (recoveryExecutor-2744-thread-1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1 r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:co

[...truncated too long message...]

cheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-6.3.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20161012_135246_796.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20161012_135246_796.suites -stdin
   [junit4] ERROR: JVM J0 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:773: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:709: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build.xml:252: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 229 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 177 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/177/

6 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([14DADA6311EDEE97]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicDistributedZkTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([14DADA6311EDEE97]:0)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:49922

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:49922
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:457)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:179)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
	... 47 more


FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([7030E5359048224E]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([7030E5359048224E]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:57201

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:57201
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:457)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:179)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
	... 47 more




Build Log:
[...truncated 1819 lines...]
   [junit4] Suite: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2> oct 16, 2016 5:16:46 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2>    1) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:243)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:354)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10)
   [junit4]   2>    2) Thread[id=10885, name=SUITE-TestFuzzyQuery-seed#[7030E5359048224E], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1556)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591)
   [junit4]   2>    3) Thread[id=10886, name=TEST-TestFuzzyQuery.testRandom-seed#[7030E5359048224E], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.util.WeakHashMap.getTable(WeakHashMap.java:350)
   [junit4]   2>         at java.util.WeakHashMap.getEntry(WeakHashMap.java:427)
   [junit4]   2>         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:254)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.method=testRandom -Dtests.seed=7030E5359048224E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-CR -Dtests.timezone=America/Scoresbysund -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7200s J2 | TestFuzzyQuery.testRandom <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7030E5359048224E]:0)
   [junit4]   2> oct 16, 2016 5:16:46 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> oct 16, 2016 5:17:06 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestFuzzyQuery: 
   [junit4]   2>    1) Thread[id=10886, name=TEST-TestFuzzyQuery.testRandom-seed#[7030E5359048224E], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> oct 16, 2016 5:17:06 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=10886, name=TEST-TestFuzzyQuery.testRandom-seed#[7030E5359048224E], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2> oct 16, 2016 5:17:09 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=10886, name=TEST-TestFuzzyQuery.testRandom-seed#[7030E5359048224E], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Object.hashCode(Native Method)
   [junit4]   2>         at java.util.WeakHashMap.hash(WeakHashMap.java:298)
   [junit4]   2>         at java.util.WeakHashMap.getEntry(WeakHashMap.java:426)
   [junit4]   2>         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:254)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62), sim=RandomSimilarity(queryNorm=false,coord=crazy): {field=DFR I(F)B2}, locale=es-CR, timezone=America/Scoresbysund
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=195032264,total=530055168
   [junit4]   2> NOTE: All tests run in this JVM: [TestRollback, TestCharTermAttributeImpl, TestDocBoost, TestLucene53NormsFormat, TestCharsRef, TestTermsEnum2, TestNamedSPILoader, TestNoDeletionPolicy, TestCollectionUtil, TestRecyclingIntBlockAllocator, TestReqExclBulkScorer, TestMultiMMap, TestLucene50StoredFieldsFormatHighCompression, TestNeedsScores, TestBinaryDocValuesUpdates, TestForUtil, TestUTF32ToUTF8, TestTimeLimitingCollector, TestElevationComparator, TestNRTThreads, TestConcurrentMergeScheduler, TestByteBlockPool, TestVirtualMethod, TestSortedNumericSortField, TestNorms, TestPayloadsOnVectors, TestMultiTermsEnum, TestLRUQueryCache, TestSpanNearQuery, TestLongPostings, TestReaderWrapperDVTypeCheck, TestLongBitSet, TestIndexWriterReader, TestCodecHoldsOpenFiles, TestLazyProxSkipping, TestSizeBoundedForceMerge, TestParallelTermEnum, TestNoMergePolicy, TestMixedCodecs, TestTermVectorsWriter, Test2BPagedBytes, TestIndexWriterForceMerge, TestBooleanMinShouldMatch, TestSparseFixedBitDocIdSet, TestGeoUtils, TestBlockPostingsFormat, TestFieldValueQuery, TestDirectoryReader, TestAtomicUpdate, TestForceMergeForever, TestWordlistLoader, TestRollingUpdates, TestFrequencyTrackingRingBuffer, TestOrdinalMap, TestIndexCommit, TestSpanTermQuery, TestCustomSearcherSort, TestFileSwitchDirectory, TestTrackingDirectoryWrapper, TestSimpleFSLockFactory, TestBinaryDocument, TestSynonymQuery, TestDocIdSetBuilder, FiniteStringsIteratorTest, TestMultiLevelSkipList, TestMultiValuedNumericRangeQuery, TestStressNRT, TestSameScoresWithThreads, TestDocValuesRewriteMethod, TestComplexExplanationsOfNonMatches, TestSpanSearchEquivalence, TestAssertions, TestAutomatonQuery, TestFilterDirectoryReader, TestIndexWriter, TestIndexWriterMerging, TestBytesStore, TestPackedInts, TestNumericRangeQuery32, TestAddIndexes, TestShardSearching, TestIndexWriterExceptions, TestDeletionPolicy, TestBooleanOr, TestBasics, TestSloppyPhraseQuery, TestIndexWriterDelete, TestIndexWriterMergePolicy, TestPersistentSnapshotDeletionPolicy, TestDirectoryReaderReopen, TestSearchWithThreads, TestHighCompressionMode, TestDocumentsWriterStallControl, TestStressIndexing, TestSnapshotDeletionPolicy, TestTieredMergePolicy, TestPerFieldPostingsFormat, TestArrayUtil, TestLevenshteinAutomata, TestBufferedIndexInput, TestTermsEnum, TestConsistentFieldNumbers, TestWeakIdentityMap, TestPhraseQuery, TestBagOfPositions, TestNRTCachingDirectory, TestParallelCompositeReader, TestIndexWriterNRTIsCurrent, TestPerFieldPostingsFormat2, TestSimilarityBase, TestLegacyNumericUtils, TestDeterminizeLexicon, Test2BDocs, TestSpans, TestSpanExplanations, TestFastCompressionMode, TestMultiPhraseQuery, TestDeterminism, TestMergeSchedulerExternal, TestCodecs, TestOmitPositions, TestParallelLeafReader, TestBytesRefHash, TestIndexInput, TestDisjunctionMaxQuery, TestOmitTf, TestNearSpansOrdered, TestTermRangeQuery, TestDocumentsWriterDeleteQueue, TestDocumentWriter, TestCrashCausesCorruptIndex, TestScorerPerf, TestMultiTermConstantScore, TestLiveFieldValues, TestFuzzyQuery]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.seed=7030E5359048224E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=es-CR -Dtests.timezone=America/Scoresbysund -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestFuzzyQuery (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7030E5359048224E]:0)
   [junit4] Completed [439/439 (1!)] on J2 in 7223.14s, 9 tests, 2 errors <<< FAILURES!

[...truncated 9769 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/init-core-data-001
   [junit4]   2> 307395 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[14DADA6311EDEE97]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 307395 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[14DADA6311EDEE97]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 307397 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 307397 INFO  (Thread-807) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 307397 INFO  (Thread-807) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 307498 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.ZkTestServer start zk server on port:37038
   [junit4]   2> 307517 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 307520 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 307522 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 307523 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 307525 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 307526 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 307529 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 307531 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 307532 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 307548 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 307550 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 307724 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/control-001/cores/collection1
   [junit4]   2> 307732 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 307756 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1e698bce{/,null,AVAILABLE}
   [junit4]   2> 307771 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.ServerConnector Started ServerConnector@1039e5c9{SSL,[ssl, http/1.1]}{127.0.0.1:36795}
   [junit4]   2> 307771 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.Server Started @310950ms
   [junit4]   2> 307771 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/tempDir-001/control/data, hostContext=/, hostPort=36795, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/control-001/cores}
   [junit4]   2> 307771 ERROR (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 307771 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 307771 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 307771 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 307771 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-16T19:28:54.660Z
   [junit4]   2> 307784 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 307784 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/control-001/solr.xml
   [junit4]   2> 307800 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 307801 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37038/solr
   [junit4]   2> 307869 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:36795_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36795_
   [junit4]   2> 307870 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:36795_    ] o.a.s.c.Overseer Overseer (id=96773481056894981-127.0.0.1:36795_-n_0000000000) starting
   [junit4]   2> 307896 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:36795_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36795_
   [junit4]   2> 307899 INFO  (zkCallback-523-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 307911 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:36795_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/control-001/cores
   [junit4]   2> 307911 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:36795_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 307915 INFO  (OverseerStateUpdate-96773481056894981-127.0.0.1:36795_-n_0000000000) [n:127.0.0.1:36795_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 308929 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 308961 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 309119 WARN  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 309121 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 309134 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 309134 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/control-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/control-001/cores/collection1/data/]
   [junit4]   2> 309134 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ffd13fa
   [junit4]   2> 309135 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 309140 WARN  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 309149 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 309149 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 309150 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 309150 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 309150 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1f7878f9
   [junit4]   2> 309150 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4d8ffa6b[collection1] main]
   [junit4]   2> 309152 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 309152 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 309152 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 309153 INFO  (searcherExecutor-1304-thread-1-processing-n:127.0.0.1:36795_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4d8ffa6b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 309153 INFO  (coreLoadExecutor-1303-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1548375698746376192
   [junit4]   2> 309164 INFO  (coreZkRegister-1296-thread-1-processing-n:127.0.0.1:36795_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 309164 INFO  (coreZkRegister-1296-thread-1-processing-n:127.0.0.1:36795_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 309164 INFO  (coreZkRegister-1296-thread-1-processing-n:127.0.0.1:36795_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:36795/collection1/
   [junit4]   2> 309164 INFO  (coreZkRegister-1296-thread-1-processing-n:127.0.0.1:36795_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 309164 INFO  (coreZkRegister-1296-thread-1-processing-n:127.0.0.1:36795_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:36795/collection1/ has no replicas
   [junit4]   2> 309169 INFO  (coreZkRegister-1296-thread-1-processing-n:127.0.0.1:36795_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:36795/collection1/ shard1
   [junit4]   2> 309321 INFO  (coreZkRegister-1296-thread-1-processing-n:127.0.0.1:36795_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:36795_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 309417 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 309419 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 309520 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-1-001/cores/collection1
   [junit4]   2> 309520 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-1-001
   [junit4]   2> 309525 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 309527 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5c9168d0{/,null,AVAILABLE}
   [junit4]   2> 309530 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.ServerConnector Started ServerConnector@7f8c52a1{SSL,[ssl, http/1.1]}{127.0.0.1:49922}
   [junit4]   2> 309530 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.Server Started @312709ms
   [junit4]   2> 309531 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=49922, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-1-001/cores}
   [junit4]   2> 309531 ERROR (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 309531 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 309531 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 309531 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 309531 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-16T19:28:56.420Z
   [junit4]   2> 309534 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 309534 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-1-001/solr.xml
   [junit4]   2> 309541 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 309542 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37038/solr
   [junit4]   2> 309552 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:49922_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 309557 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:49922_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49922_
   [junit4]   2> 309559 INFO  (zkCallback-523-thread-2-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 309559 INFO  (zkCallback-527-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 309567 INFO  (zkCallback-532-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 309573 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:49922_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-1-001/cores
   [junit4]   2> 309573 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:49922_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 309576 INFO  (OverseerStateUpdate-96773481056894981-127.0.0.1:36795_-n_0000000000) [n:127.0.0.1:36795_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 310589 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 310603 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 310691 WARN  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 310693 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 310705 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 310705 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-1-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 310705 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ffd13fa
   [junit4]   2> 310706 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 310712 WARN  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 310723 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 310723 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 310724 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 310724 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 310725 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2f97838f
   [junit4]   2> 310725 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@20765738[collection1] main]
   [junit4]   2> 310726 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 310727 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 310727 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 310728 INFO  (searcherExecutor-1315-thread-1-processing-n:127.0.0.1:49922_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@20765738[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 310729 INFO  (coreLoadExecutor-1314-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1548375700398931968
   [junit4]   2> 310737 INFO  (coreZkRegister-1309-thread-1-processing-n:127.0.0.1:49922_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 310737 INFO  (coreZkRegister-1309-thread-1-processing-n:127.0.0.1:49922_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 310737 INFO  (coreZkRegister-1309-thread-1-processing-n:127.0.0.1:49922_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:49922/collection1/
   [junit4]   2> 310737 INFO  (coreZkRegister-1309-thread-1-processing-n:127.0.0.1:49922_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 310737 INFO  (coreZkRegister-1309-thread-1-processing-n:127.0.0.1:49922_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:49922/collection1/ has no replicas
   [junit4]   2> 310743 INFO  (coreZkRegister-1309-thread-1-processing-n:127.0.0.1:49922_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:49922/collection1/ shard2
   [junit4]   2> 310894 INFO  (coreZkRegister-1309-thread-1-processing-n:127.0.0.1:49922_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:49922_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 311182 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-2-001/cores/collection1
   [junit4]   2> 311183 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-2-001
   [junit4]   2> 311185 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 311187 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6833c0f7{/,null,AVAILABLE}
   [junit4]   2> 311192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.ServerConnector Started ServerConnector@fa5ee5d{SSL,[ssl, http/1.1]}{127.0.0.1:54640}
   [junit4]   2> 311192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.Server Started @314370ms
   [junit4]   2> 311192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=54640, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-2-001/cores}
   [junit4]   2> 311192 ERROR (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 311192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 311192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 311192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 311192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-16T19:28:58.081Z
   [junit4]   2> 311195 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 311195 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-2-001/solr.xml
   [junit4]   2> 311202 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 311202 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37038/solr
   [junit4]   2> 311216 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:54640_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 311222 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:54640_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54640_
   [junit4]   2> 311224 INFO  (zkCallback-527-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 311224 INFO  (zkCallback-532-thread-1-processing-n:127.0.0.1:49922_) [n:127.0.0.1:49922_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 311225 INFO  (zkCallback-523-thread-1-processing-n:127.0.0.1:36795_) [n:127.0.0.1:36795_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 311233 INFO  (zkCallback-538-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 311237 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:54640_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-2-001/cores
   [junit4]   2> 311237 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:54640_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 311240 INFO  (OverseerStateUpdate-96773481056894981-127.0.0.1:36795_-n_0000000000) [n:127.0.0.1:36795_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 312254 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 312268 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 312360 WARN  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 312362 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 312378 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 312379 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-2-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 312379 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ffd13fa
   [junit4]   2> 312382 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 312390 WARN  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 312409 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 312409 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 312410 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 312410 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 312411 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@5374814e
   [junit4]   2> 312412 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@29ee7748[collection1] main]
   [junit4]   2> 312413 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 312414 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 312414 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 312415 INFO  (searcherExecutor-1326-thread-1-processing-n:127.0.0.1:54640_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@29ee7748[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 312415 INFO  (coreLoadExecutor-1325-thread-1-processing-n:127.0.0.1:54640_) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1548375702166831104
   [junit4]   2> 312424 INFO  (coreZkRegister-1320-thread-1-processing-n:127.0.0.1:54640_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 312424 INFO  (coreZkRegister-1320-thread-1-processing-n:127.0.0.1:54640_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 312424 INFO  (coreZkRegister-1320-thread-1-processing-n:127.0.0.1:54640_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:54640/collection1/
   [junit4]   2> 312424 INFO  (coreZkRegister-1320-thread-1-processing-n:127.0.0.1:54640_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 312424 INFO  (coreZkRegister-1320-thread-1-processing-n:127.0.0.1:54640_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:54640/collection1/ has no replicas
   [junit4]   2> 312429 INFO  (coreZkRegister-1320-thread-1-processing-n:127.0.0.1:54640_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:54640/collection1/ shard1
   [junit4]   2> 312581 INFO  (coreZkRegister-1320-thread-1-processing-n:127.0.0.1:54640_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:54640_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 312842 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-3-001/cores/collection1
   [junit4]   2> 312843 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-3-001
   [junit4]   2> 312846 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 312848 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@26063655{/,null,AVAILABLE}
   [junit4]   2> 312851 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.ServerConnector Started ServerConnector@516204e4{SSL,[ssl, http/1.1]}{127.0.0.1:41887}
   [junit4]   2> 312852 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.e.j.s.Server Started @316031ms
   [junit4]   2> 312852 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=41887, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-3-001/cores}
   [junit4]   2> 312852 ERROR (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 312852 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.3.0
   [junit4]   2> 312852 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 312853 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 312853 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-16T19:28:59.742Z
   [junit4]   2> 312857 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 312857 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_14DADA6311EDEE97-001/shard-3-001/solr.xml
   [junit4]   2> 312866 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 312867 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37038/solr
   [junit4]   2> 312879 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:41887_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 312885 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[14DADA6311EDEE97]) [n:127.0.0.1:41887_    ] o.a.s.c.ZkController Regis

[...truncated too long message...]

    at org.apache.solr.cloud.ChaosMonkey.stop(ChaosMonkey.java:574)
   [junit4]   2>         at org.apache.solr.cloud.AbstractFullDistribZkTestBase.destroyServers(AbstractFullDistribZkTestBase.java:1540)
   [junit4]   2>         at org.apache.solr.BaseDistributedSearchTestCase.distribTearDown(BaseDistributedSearchTestCase.java:314)
   [junit4]   2>         at org.apache.solr.cloud.AbstractDistribZkTestBase.distribTearDown(AbstractDistribZkTestBase.java:283)
   [junit4]   2>         at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribTearDown(AbstractFullDistribZkTestBase.java:1514)
   [junit4]   2>         at org.apache.solr.cloud.BasicDistributedZkTest.distribTearDown(BasicDistributedZkTest.java:1142)
   [junit4]   2>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:969)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>   10) Thread[id=16011, name=org.eclipse.jetty.server.session.HashSessionManager@ed463a2Timer, state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]   2>         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]   2>         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>   11) Thread[id=16024, name=Thread-7961, state=WAITING, group=TGRP-BasicDistributedZkTest]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Object.wait(Object.java:502)
   [junit4]   2>         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1274)
   [junit4]   2>   12) Thread[id=16020, name=TEST-BasicDistributedZkTest.test-seed#[14DADA6311EDEE97]-SendThread(127.0.0.1:51086), state=TIMED_WAITING, group=TGRP-BasicDistributedZkTest]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=ClassicSimilarity, locale=tr, timezone=Asia/Thimphu
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=20,free=140254160,total=528482304
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsUnloadDistributedZkTest, OverseerStatusTest, CurrencyFieldXmlFileTest, SimpleFacetsTest, ResponseLogComponentTest, AddSchemaFieldsUpdateProcessorFactoryTest, TolerantUpdateProcessorTest, TestSchemaSimilarityResource, RecoveryAfterSoftCommitTest, TestExtendedDismaxParser, BadComponentTest, DistribCursorPagingTest, CdcrUpdateLogTest, DistributedFacetPivotSmallAdvancedTest, TestCloudPivotFacet, TestZkChroot, TestDFRSimilarityFactory, StatelessScriptUpdateProcessorFactoryTest, TestAuthenticationFramework, UpdateParamsTest, CloudMLTQParserTest, HLLSerializationTest, TestLRUStatsCache, TestDefaultSearchFieldResource, TestReqParamsAPI, SolrTestCaseJ4Test, TestPostingsSolrHighlighter, SuggesterWFSTTest, TestLeaderElectionZkExpiry, SharedFSAutoReplicaFailoverTest, TestMacros, TestXmlQParser, SolrCoreCheckLockOnStartupTest, TestConfigSetsAPI, LeaderElectionTest, BlockDirectoryTest, SOLR749Test, TestSolrJ, OverseerTest, StatsReloadRaceTest, TestFreeTextSuggestions, HdfsLockFactoryTest, CoreSorterTest, TestUseDocValuesAsStored, TestWriterPerf, TestPivotHelperCode, TestUtils, TestLMJelinekMercerSimilarityFactory, DirectUpdateHandlerTest, TestLFUCache, TestUninvertingReader, TestConfigOverlay, TestDistribIDF, DistributedQueryComponentCustomSortTest, XmlUpdateRequestHandlerTest, FacetPivotSmallTest, BinaryUpdateRequestHandlerTest, TestSmileRequest, TestExceedMaxTermLength, TestReload, TestSolrCoreSnapshots, PreAnalyzedFieldTest, PingRequestHandlerTest, TestSolrQueryResponse, TestDFISimilarityFactory, TestAnalyzeInfixSuggestions, LukeRequestHandlerTest, TestConfig, JSONWriterTest, QueryParsingTest, TestAuthorizationFramework, BitVectorTest, TestManagedStopFilterFactory, TestIndexingPerformance, TestTestInjection, BlockJoinFacetRandomTest, CheckHdfsIndexTest, HttpPartitionTest, TestClusterStateMutator, TestSQLHandlerNonCloud, TestFieldCacheSortRandom, TestGraphTermsQParserPlugin, CdcrBootstrapTest, AutoCommitTest, TestJsonFacets, LeaderInitiatedRecoveryOnCommitTest, SuggestComponentTest, StatsComponentTest, CoreAdminCreateDiscoverTest, TestCoreDiscovery, ShowFileRequestHandlerTest, TestRandomFlRTGCloud, DocValuesMultiTest, HdfsSyncSliceTest, RecoveryZkTest, TestBulkSchemaAPI, TestMaxScoreQueryParser, TestGroupingSearch, BasicDistributedZkTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZkTest -Dtests.seed=14DADA6311EDEE97 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=tr -Dtests.timezone=Asia/Thimphu -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | BasicDistributedZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([14DADA6311EDEE97]:0)
   [junit4] Completed [640/640 (3!)] on J2 in 7223.90s, 1 test, 2 errors <<< FAILURES!

[...truncated 60338 lines...]



[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 176 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/176/

7 tests failed.
FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([54785754D624E476]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([54785754D624E476]:0)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testOps

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([BAADB79CF21D9217:A6D52B7FBD2E99CA]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:795)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testOps(CdcrReplicationDistributedZkTest.java:463)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:53231/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:53231/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1529)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:664)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
	... 49 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
	at __randomizedtesting.SeedInfo.seed([BAADB79CF21D9217:32F988465CE1FFEF]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:309)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.TestDynamicLoading.testDynamicLoading

Error Message:
Could not get expected value  'X val' for path 'x' full output: {   "responseHeader":{     "status":0,     "QTime":0},   "params":{"wt":"json"},   "context":{     "webapp":"",     "path":"/test1",     "httpMethod":"GET"},   "class":"org.apache.solr.core.BlobStoreTestRequestHandler",   "x":null},  from server:  null

Stack Trace:
java.lang.AssertionError: Could not get expected value  'X val' for path 'x' full output: {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "params":{"wt":"json"},
  "context":{
    "webapp":"",
    "path":"/test1",
    "httpMethod":"GET"},
  "class":"org.apache.solr.core.BlobStoreTestRequestHandler",
  "x":null},  from server:  null
	at __randomizedtesting.SeedInfo.seed([BAADB79CF21D9217:62E09ACB05C037B7]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:535)
	at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:232)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:60028

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:60028
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:515)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:179)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
	... 47 more




Build Log:
[...truncated 1836 lines...]
   [junit4] Suite: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2> Oct 15, 2016 6:33:30 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2>    1) Thread[id=4075, name=SUITE-TestFuzzyQuery-seed#[54785754D624E476], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1556)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591)
   [junit4]   2>    2) Thread[id=4076, name=TEST-TestFuzzyQuery.testRandom-seed#[54785754D624E476], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Object.hashCode(Native Method)
   [junit4]   2>         at java.util.WeakHashMap.hash(WeakHashMap.java:298)
   [junit4]   2>         at java.util.WeakHashMap.get(WeakHashMap.java:396)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getPerThread(RandomizedContext.java:315)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:110)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:132)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    3) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:243)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:354)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10)
   [junit4]   2>    4) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.method=testRandom -Dtests.seed=54785754D624E476 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-GB -Dtests.timezone=America/Fort_Wayne -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7200s J1 | TestFuzzyQuery.testRandom <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54785754D624E476]:0)
   [junit4]   2> Oct 15, 2016 6:33:30 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Oct 15, 2016 6:33:50 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestFuzzyQuery: 
   [junit4]   2>    1) Thread[id=4076, name=TEST-TestFuzzyQuery.testRandom-seed#[54785754D624E476], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.util.WeakHashMap.getTable(WeakHashMap.java:350)
   [junit4]   2>         at java.util.WeakHashMap.getEntry(WeakHashMap.java:427)
   [junit4]   2>         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:254)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Oct 15, 2016 6:33:50 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=4076, name=TEST-TestFuzzyQuery.testRandom-seed#[54785754D624E476], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2> Oct 15, 2016 6:33:53 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=4076, name=TEST-TestFuzzyQuery.testRandom-seed#[54785754D624E476], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Object.hashCode(Native Method)
   [junit4]   2>         at java.util.WeakHashMap.hash(WeakHashMap.java:298)
   [junit4]   2>         at java.util.WeakHashMap.getEntry(WeakHashMap.java:426)
   [junit4]   2>         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:254)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:503)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=1, maxDocsPerChunk=2, blockSize=723), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=1, blockSize=723)), sim=ClassicSimilarity, locale=en-GB, timezone=America/Fort_Wayne
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=209655248,total=250609664
   [junit4]   2> NOTE: All tests run in this JVM: [TestFlex, TestDirectory, TestReadOnlyIndex, TestTopDocsMerge, TestPolygon2D, TestMultiDocValues, TestFixedLengthBytesRefArray, TestConjunctions, TestByteArrayDataInput, TestApproximationSearchEquivalence, TestIndexWriterOnDiskFull, TestSegmentReader, TestTotalHitCountCollector, TestSubScorerFreqs, TestSortedSetDocValues, TestDocumentsWriterDeleteQueue, TestPhrasePrefixQuery, TestPerFieldDocValuesFormat, TestFieldReuse, TestSnapshotDeletionPolicy, TestFixedBitSet, TestNumericTokenStream, TestSearch, TestTermVectors, TestDirectPacked, TestPackedTokenAttributeImpl, Test2BPositions, TestSimilarityBase, TestPerFieldPostingsFormat2, TestFilterSpans, TestPerFieldPostingsFormat, TestNumericDocValuesUpdates, TestCheckIndex, TestNIOFSDirectory, TestPrefixRandom, TestAutomaton, TestMinShouldMatch2, TestTerm, TestBooleanQuery, TestSearchWithThreads, TestNoMergeScheduler, TestBooleanRewrites, TestMutablePointsReaderUtils, TestAllFilesHaveChecksumFooter, TestSpanContainQuery, TestParallelReaderEmptyIndex, TestMathUtil, TestSegmentMerger, TestStopFilter, TestSortRandom, TestRollingBuffer, TestMultiset, TestIndexWriterExceptions, TestOmitTf, TestBagOfPostings, TestPagedBytes, Test2BSortedDocValuesOrds, TestSleepingLockWrapper, TestIndexableField, TestIndexSorting, TestIntArrayDocIdSet, TestDocsAndPositions, TestUsageTrackingFilterCachingPolicy, TestField, TestIndexWriterUnicode, TestPackedInts, TestDuelingCodecs, TestPriorityQueue, TestScoreCachingWrappingScorer, TestSortRescorer, TestBlockPostingsFormat3, TestSpanOrQuery, TestLiveFieldValues, TestOmitNorms, TestComplexExplanations, TestRegexpRandom2, TestNRTCachingDirectory, TestFastDecompressionMode, TestTransactions, TestCrash, TestMultiThreadTermVectors, TestSegmentTermDocs, TestSimpleSearchEquivalence, TestCustomSearcherSort, TestHugeRamFile, TestBooleanMinShouldMatch, TestLegacyNumericUtils, TestDeterminizeLexicon, Test2BDocs, TestPostingsOffsets, TestMixedCodecs, TestTransactionRollback, TestUTF32ToUTF8, TestFastCompressionMode, TestStressIndexing2, TestRAMDirectory, TestMultiPhraseQuery, TestLazyProxSkipping, TestDeterminism, TestMergeSchedulerExternal, TestCharTermAttributeImpl, TestParallelLeafReader, TestBytesRefHash, TestIndexInput, TestSloppyPhraseQuery2, TestForceMergeForever, TestNearSpansOrdered, TestMultiFields, TestTermRangeQuery, TestCrashCausesCorruptIndex, TestScorerPerf, TestForTooMuchCloning, TestFuzzyQuery]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.seed=54785754D624E476 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=en-GB -Dtests.timezone=America/Fort_Wayne -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestFuzzyQuery (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54785754D624E476]:0)
   [junit4] Completed [439/439 (1!)] on J1 in 7223.22s, 11 tests, 2 errors <<< FAILURES!

[...truncated 9453 lines...]
   [junit4] Suite: org.apache.solr.core.TestDynamicLoading
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/init-core-data-001
   [junit4]   2> 6859 INFO  (SUITE-TestDynamicLoading-seed#[BAADB79CF21D9217]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 6860 INFO  (SUITE-TestDynamicLoading-seed#[BAADB79CF21D9217]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 6887 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 6909 INFO  (Thread-11) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 6912 INFO  (Thread-11) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 7105 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.ZkTestServer start zk server on port:39130
   [junit4]   2> 7271 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 8714 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x157caf1a4fc0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 8783 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 8809 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 8818 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 8824 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 8828 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 8832 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 8836 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 8841 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 8852 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 8858 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 8862 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 9634 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/control-001/cores/collection1
   [junit4]   2> 9637 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 9662 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2b7663a7{/,null,AVAILABLE}
   [junit4]   2> 9668 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.ServerConnector Started ServerConnector@39f74fa5{HTTP/1.1,[http/1.1]}{127.0.0.1:45809}
   [junit4]   2> 9668 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.Server Started @12927ms
   [junit4]   2> 9668 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/tempDir-001/control/data, hostContext=/, hostPort=45809, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/control-001/cores}
   [junit4]   2> 9669 ERROR (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 9669 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.3.0
   [junit4]   2> 9669 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 9670 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 9670 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-16T00:43:39.177Z
   [junit4]   2> 9670 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 9765 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 9765 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/control-001/solr.xml
   [junit4]   2> 9808 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 9817 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39130/solr
   [junit4]   2> 10149 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:45809_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45809_
   [junit4]   2> 10154 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:45809_    ] o.a.s.c.Overseer Overseer (id=96769056215662596-127.0.0.1:45809_-n_0000000000) starting
   [junit4]   2> 10396 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:45809_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45809_
   [junit4]   2> 10411 INFO  (zkCallback-9-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 10551 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:45809_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/control-001/cores
   [junit4]   2> 10551 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:45809_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 10684 INFO  (OverseerStateUpdate-96769056215662596-127.0.0.1:45809_-n_0000000000) [n:127.0.0.1:45809_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 11786 WARN  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 11787 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 11924 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 12515 WARN  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 12533 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 12627 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 12628 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/control-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/control-001/cores/collection1/data/]
   [junit4]   2> 12628 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@47ed205
   [junit4]   2> 12673 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=29.78515625, floorSegmentMB=0.548828125, forceMergeDeletesPctAllowed=28.388270322180833, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 12730 WARN  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 12808 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 12814 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 12833 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 12833 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 12834 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 12851 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@213f62c4[collection1] main]
   [junit4]   2> 12856 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 12865 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 12865 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 12870 INFO  (searcherExecutor-22-thread-1-processing-n:127.0.0.1:45809_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@213f62c4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 12872 INFO  (coreLoadExecutor-21-thread-1-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1548304905539682304
   [junit4]   2> 12953 INFO  (coreZkRegister-14-thread-1-processing-n:127.0.0.1:45809_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 12953 INFO  (coreZkRegister-14-thread-1-processing-n:127.0.0.1:45809_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 12953 INFO  (coreZkRegister-14-thread-1-processing-n:127.0.0.1:45809_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45809/collection1/
   [junit4]   2> 12954 INFO  (coreZkRegister-14-thread-1-processing-n:127.0.0.1:45809_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 12954 INFO  (coreZkRegister-14-thread-1-processing-n:127.0.0.1:45809_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:45809/collection1/ has no replicas
   [junit4]   2> 13005 INFO  (coreZkRegister-14-thread-1-processing-n:127.0.0.1:45809_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45809/collection1/ shard1
   [junit4]   2> 13164 INFO  (coreZkRegister-14-thread-1-processing-n:127.0.0.1:45809_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45809_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 13181 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 13186 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 13186 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
   [junit4]   2> 13471 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-1-001/cores/collection1
   [junit4]   2> 13472 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-1-001
   [junit4]   2> 13474 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 13553 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6a462ef9{/,null,AVAILABLE}
   [junit4]   2> 13554 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.ServerConnector Started ServerConnector@5800c123{HTTP/1.1,[http/1.1]}{127.0.0.1:51344}
   [junit4]   2> 13555 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.Server Started @16813ms
   [junit4]   2> 13555 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=51344, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-1-001/cores}
   [junit4]   2> 13555 ERROR (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 13556 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.3.0
   [junit4]   2> 13556 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 13557 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 13557 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-16T00:43:43.064Z
   [junit4]   2> 13569 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 13570 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-1-001/solr.xml
   [junit4]   2> 13609 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 13617 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39130/solr
   [junit4]   2> 13707 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:51344_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 13722 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:51344_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51344_
   [junit4]   2> 13724 INFO  (zkCallback-13-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 13732 INFO  (zkCallback-9-thread-2-processing-n:127.0.0.1:45809_) [n:127.0.0.1:45809_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 13732 INFO  (zkCallback-19-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 13777 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:51344_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-1-001/cores
   [junit4]   2> 13777 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [n:127.0.0.1:51344_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 13807 INFO  (OverseerStateUpdate-96769056215662596-127.0.0.1:45809_-n_0000000000) [n:127.0.0.1:45809_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 13926 INFO  (zkCallback-19-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 14911 WARN  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 14935 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 15087 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 15403 WARN  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 15410 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 15463 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 15463 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-1-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 15464 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@47ed205
   [junit4]   2> 15468 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=29.78515625, floorSegmentMB=0.548828125, forceMergeDeletesPctAllowed=28.388270322180833, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 15505 WARN  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 15604 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 15604 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 15605 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 15605 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 15613 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 15619 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@797819a3[collection1] main]
   [junit4]   2> 15630 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 15632 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 15632 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 15650 INFO  (searcherExecutor-33-thread-1-processing-n:127.0.0.1:51344_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@797819a3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 15650 INFO  (coreLoadExecutor-32-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1548304908452626432
   [junit4]   2> 15678 INFO  (coreZkRegister-27-thread-1-processing-n:127.0.0.1:51344_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 15679 INFO  (coreZkRegister-27-thread-1-processing-n:127.0.0.1:51344_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 15679 INFO  (coreZkRegister-27-thread-1-processing-n:127.0.0.1:51344_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51344/collection1/
   [junit4]   2> 15679 INFO  (coreZkRegister-27-thread-1-processing-n:127.0.0.1:51344_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 15679 INFO  (coreZkRegister-27-thread-1-processing-n:127.0.0.1:51344_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:51344/collection1/ has no replicas
   [junit4]   2> 15683 INFO  (zkCallback-19-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 15700 INFO  (coreZkRegister-27-thread-1-processing-n:127.0.0.1:51344_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:51344/collection1/ shard2
   [junit4]   2> 15808 INFO  (zkCallback-19-thread-1-processing-n:127.0.0.1:51344_) [n:127.0.0.1:51344_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 15951 INFO  (coreZkRegister-27-thread-1-processing-n:127.0.0.1:51344_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51344_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 16054 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-2-001/cores/collection1
   [junit4]   2> 16055 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading_BAADB79CF21D9217-001/shard-2-001
   [junit4]   2> 16056 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 16060 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1c36b8b9{/,null,AVAILABLE}
   [junit4]   2> 16061 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.ServerConnector Started ServerConnector@576b2a11{HTTP/1.1,[http/1.1]}{127.0.0.1:48000}
   [junit4]   2> 16061 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.e.j.s.Server Started @19320ms
   [junit4]   2> 16061 INFO  (TEST-TestDynamicLoading.testDynamicLoading-seed#[BAADB79CF21D9217]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTe

[...truncated too long message...]

java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:52)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 5243274 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[BAADB79CF21D9217]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_BAADB79CF21D9217-001
   [junit4]   2> Oct 16, 2016 2:10:53 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62), sim=ClassicSimilarity, locale=cs-CZ, timezone=Pacific/Chatham
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=127001064,total=490733568
   [junit4]   2> NOTE: All tests run in this JVM: [CheckHdfsIndexTest, StatelessScriptUpdateProcessorFactoryTest, TestRecovery, SolrPluginUtilsTest, OverseerCollectionConfigSetProcessorTest, MultiThreadedOCPTest, BlockDirectoryTest, LukeRequestHandlerTest, HdfsSyncSliceTest, SimpleFacetsTest, CoreAdminHandlerTest, TestIndexSearcher, TestOverriddenPrefixQueryForCustomFieldType, CdcrUpdateLogTest, TestSortByMinMaxFunction, TestSolrCoreSnapshots, CdcrRequestHandlerTest, DistributedFacetPivotSmallAdvancedTest, TestFieldCacheReopen, TestSchemaSimilarityResource, DebugComponentTest, TestCustomSort, SuggestComponentTest, TestTolerantSearch, TestCloudPivotFacet, CollectionsAPISolrJTest, CloudMLTQParserTest, TestXIncludeConfig, TolerantUpdateProcessorTest, LeaderElectionTest, ResponseLogComponentTest, BlockJoinFacetRandomTest, DeleteInactiveReplicaTest, XmlUpdateRequestHandlerTest, ConnectionReuseTest, SSLMigrationTest, BasicDistributedZk2Test, TestCoreDiscovery, CdcrBootstrapTest, TestExceedMaxTermLength, TestStressLiveNodes, CurrencyFieldXmlFileTest, TestJmxMonitoredMap, FacetPivotSmallTest, ConnectionManagerTest, DeleteReplicaTest, TestSQLHandlerNonCloud, BitVectorTest, UnloadDistributedZkTest, TestSort, OpenExchangeRatesOrgProviderTest, TestLRUStatsCache, TestManagedStopFilterFactory, TestLocalFSCloudBackupRestore, TestWriterPerf, TestSolrCloudWithDelegationTokens, HardAutoCommitTest, StatsComponentTest, DocValuesMultiTest, BadIndexSchemaTest, HdfsBasicDistributedZk2Test, TestAuthenticationFramework, TestLeaderElectionZkExpiry, SharedFSAutoReplicaFailoverTest, PrimitiveFieldTypeTest, TestLFUCache, BlobRepositoryCloudTest, SpellPossibilityIteratorTest, TestSolrQueryParser, DistributedQueueTest, TlogReplayBufferedWhileIndexingTest, TestSolrConfigHandler, TestPostingsSolrHighlighter, TestBulkSchemaConcurrent, CoreSorterTest, DistributedFacetExistsSmallTest, TestReplicaProperties, HighlighterMaxOffsetTest, TestConfigSets, TestRandomFlRTGCloud, ShowFileRequestHandlerTest, HdfsChaosMonkeySafeLeaderTest, CoreAdminCreateDiscoverTest, QueryParsingTest, TestReqParamsAPI, HdfsLockFactoryTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestLMJelinekMercerSimilarityFactory, FieldAnalysisRequestHandlerTest, TestDFRSimilarityFactory, TestReloadDeadlock, TestObjectReleaseTracker, TimeZoneUtilsTest, SuggesterTest, CollectionStateFormat2Test, DirectUpdateHandlerTest, BooleanFieldTest, TestSolrQueryResponse, TestCryptoKeys, TestReversedWildcardFilterFactory, ZkSolrClientTest, TestRandomDVFaceting, AutoCommitTest, TestDistributedGrouping, TestHashPartitioner, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestRealTimeGet, TestStressReorder, TestJoin, TestReload, TestCoreContainer, TestSolr4Spatial, SpellCheckComponentTest, TestGroupingSearch, QueryElevationComponentTest, ConvertedLegacyTest, TestFunctionQuery, TestLazyCores, TestBadConfig, CurrencyFieldOpenExchangeTest, TestExtendedDismaxParser, SuggesterFSTTest, SolrRequestParserTest, SuggesterTSTTest, TestStressLucene, TestTrie, SuggesterWFSTTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, SortByFunctionTest, TestRemoteStreaming, TestSolrDeletionPolicy1, SolrInfoMBeanTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, StandardRequestHandlerTest, DirectSolrSpellCheckerTest, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TestOmitPositions, RequestHandlersTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, TestIndexingPerformance, RequiredFieldsTest, FastVectorHighlighterTest, LoggingHandlerTest, IndexSchemaTest, TestCollationField, UpdateRequestProcessorFactoryTest, JSONWriterTest, TestCSVResponseWriter, MBeansHandlerTest, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, TestBinaryResponseWriter, SOLR749Test, UpdateParamsTest, TestSolrIndexConfig, TestQuerySenderNoQuery, BadComponentTest, TestStressRecovery, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, TestCharFilters, TestIBSimilarityFactory, TestFastWriter, PluginInfoTest, DateMathParserTest, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestSolrJ, TestLRUCache, TestUtils, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, CircularListTest, DistributedMLTComponentTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestDistributedMissingSort, TestSimpleTrackingShardHandler, AliasIntegrationTest, AssignTest, AsyncCallRequestStatusResponseTest, CdcrVersionReplicationTest, CloudExitableDirectoryReaderTest, CollectionTooManyReplicasTest, DistribDocExpirationUpdateProcessorTest, ForceLeaderTest, LeaderFailureAfterFreshStartTest, MigrateRouteKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerStatusTest, RecoveryAfterSoftCommitTest, ReplicationFactorTest, RestartWhileUpdatingTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, SolrXmlInZkTest, TestCloudInspectUtil, TestConfigSetsAPI, TestExclusionRuleCollectionAccess, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLeaderRecoverFromLogOnStartup, TestLockTree, TestMiniSolrCloudCluster, TestSSLRandomization, TestSizeLimitedDistributedMap, TriLevelCompositeIdRoutingTest, HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [564/640 (5!)] on J2 in 510.99s, 1 test, 1 error <<< FAILURES!

[...truncated 60572 lines...]



[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 175 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/175/

5 tests failed.
FAILED:  org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField

Error Message:
this IndexWriter is closed

Stack Trace:
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
	at __randomizedtesting.SeedInfo.seed([D8EB7C81451859F1:88BA7740B2EE08E8]:0)
	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:740)
	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:754)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1558)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
	at org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField(TestDemoParallelLeafReader.java:1050)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4312)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3889)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)


FAILED:  org.apache.lucene.index.TestDemoParallelLeafReader.testRandom

Error Message:
Captured an uncaught exception in thread: Thread[id=9746, name=Lucene Merge Thread #12, state=RUNNABLE, group=TGRP-TestDemoParallelLeafReader]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=9746, name=Lucene Merge Thread #12, state=RUNNABLE, group=TGRP-TestDemoParallelLeafReader]
Caused by: org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
	at __randomizedtesting.SeedInfo.seed([D8EB7C81451859F1]:0)
	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:668)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:648)
Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
	at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:345)
	at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:696)
	at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:691)
	at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:4913)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4404)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3889)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)
Caused by: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4312)
	... 3 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)  at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:74)  at org.apache.solr.update.HdfsUpdateLog.ensureLog(HdfsUpdateLog.java:313)  at org.apache.solr.update.UpdateLog.deleteByQuery(UpdateLog.java:490)  at org.apache.solr.update.DirectUpdateHandler2.deleteByQuery(DirectUpdateHandler2.java:464)  at org.apache.solr.update.processor.RunUpdateProcessor.processDelete(RunUpdateProcessorFactory.java:78)  at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:52)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalDelete(DistributedUpdateProcessor.java:962)  at org.apache.solr.update.processor.DistributedUpdateProcessor.versionDeleteByQuery(DistributedUpdateProcessor.java:1457)  at org.apache.solr.update.processor.DistributedUpdateProcessor.doDeleteByQuery(DistributedUpdateProcessor.java:1350)  at org.apache.solr.update.processor.DistributedUpdateProcessor.processDelete(DistributedUpdateProcessor.java:1159)  at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processDelete(LogUpdateProcessorFactory.java:124)  at org.apache.solr.handler.loader.JavabinLoader.delete(JavabinLoader.java:159)  at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:112)  at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:54)  at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)  at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:153)  at org.apache.solr.core.SolrCore.execute(SolrCore.java:2088)  at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:654)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:302)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:253)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:108)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
	at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:74)
	at org.apache.solr.update.HdfsUpdateLog.ensureLog(HdfsUpdateLog.java:313)
	at org.apache.solr.update.UpdateLog.deleteByQuery(UpdateLog.java:490)
	at org.apache.solr.update.DirectUpdateHandler2.deleteByQuery(DirectUpdateHandler2.java:464)
	at org.apache.solr.update.processor.RunUpdateProcessor.processDelete(RunUpdateProcessorFactory.java:78)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:52)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalDelete(DistributedUpdateProcessor.java:962)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionDeleteByQuery(DistributedUpdateProcessor.java:1457)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doDeleteByQuery(DistributedUpdateProcessor.java:1350)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processDelete(DistributedUpdateProcessor.java:1159)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processDelete(LogUpdateProcessorFactory.java:124)
	at org.apache.solr.handler.loader.JavabinLoader.delete(JavabinLoader.java:159)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:112)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:54)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:153)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2088)
	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:654)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:302)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:253)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:108)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)


	at __randomizedtesting.SeedInfo.seed([A5006D8D59AE1922]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:261)
	at sun.reflect.GeneratedMethodAccessor28.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:50406

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:50406
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testNoCollectionSpecified(CollectionsAPIDistributedZkTest.java:579)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
	... 47 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=23695, name=collection3, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=23695, name=collection3, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:51439
	at __randomizedtesting.SeedInfo.seed([A5006D8D59AE1922]:0)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:997)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:51439
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:604)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:262)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:251)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:435)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:387)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1292)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1062)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1004)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1599)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1620)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:987)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:498)
	... 11 more




Build Log:
[...truncated 1503 lines...]
   [junit4] Suite: org.apache.lucene.index.TestDemoParallelLeafReader
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDemoParallelLeafReader -Dtests.method=testRandomMultipleSchemaGensSameField -Dtests.seed=D8EB7C81451859F1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=sk -Dtests.timezone=SystemV/PST8PDT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   21.6s J2 | TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField <<<
   [junit4]    > Throwable #1: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D8EB7C81451859F1:88BA7740B2EE08E8]:0)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:740)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:754)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1558)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField(TestDemoParallelLeafReader.java:1050)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
   [junit4]    > 	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
   [junit4]    > 	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
   [junit4]    > 	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4312)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3889)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)
   [junit4]   2> okt 13, 2016 3:01:22 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #12,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D8EB7C81451859F1]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:668)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:648)
   [junit4]   2> Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
   [junit4]   2> 	at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:345)
   [junit4]   2> 	at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:696)
   [junit4]   2> 	at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:691)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:4913)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4404)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3889)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
   [junit4]   2> 	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
   [junit4]   2> 	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
   [junit4]   2> 	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4312)
   [junit4]   2> 	... 3 more
   [junit4]   2> 
   [junit4]   2> okt 13, 2016 3:01:22 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #10,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([D8EB7C81451859F1]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:668)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:648)
   [junit4]   2> Caused by: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
   [junit4]   2> 	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
   [junit4]   2> 	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
   [junit4]   2> 	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4312)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3889)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)
   [junit4]   2> 
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDemoParallelLeafReader -Dtests.method=testRandom -Dtests.seed=D8EB7C81451859F1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/test-data/enwiki.random.lines.txt -Dtests.locale=sk -Dtests.timezone=SystemV/PST8PDT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   24.3s J2 | TestDemoParallelLeafReader.testRandom <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=9746, name=Lucene Merge Thread #12, state=RUNNABLE, group=TGRP-TestDemoParallelLeafReader]
   [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D8EB7C81451859F1]:0)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:668)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:648)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:345)
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:696)
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:691)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:4913)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4404)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3889)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)
   [junit4]    > Caused by: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
   [junit4]    > 	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
   [junit4]    > 	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
   [junit4]    > 	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4312)
   [junit4]    > 	... 3 moreThrowable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=9743, name=Lucene Merge Thread #10, state=RUNNABLE, group=TGRP-TestDemoParallelLeafReader]
   [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D8EB7C81451859F1]:0)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:668)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:648)
   [junit4]    > Caused by: java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
   [junit4]    > 	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
   [junit4]    > 	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
   [junit4]    > 	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4312)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3889)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/core/test/J2/temp/lucene.index.TestDemoParallelLeafReader_D8EB7C81451859F1-001
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=6525, maxDocsPerChunk=4, blockSize=1), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=6525, blockSize=1)), sim=ClassicSimilarity, locale=sk, timezone=SystemV/PST8PDT
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=61126024,total=519045120
   [junit4]   2> NOTE: All tests run in this JVM: [TestSimpleFSLockFactory, Test2BDocs, TestOfflineSorter, TestSegmentTermEnum, TestIndexWriterThreadsToSegments, TestToken, TestIndexCommit, TestAllFilesCheckIndexHeader, TestVersion, TestMaxTermFrequency, TestSort, TestNRTThreads, TestRecyclingIntBlockAllocator, TestSynonymQuery, TestBlendedTermQuery, TestBufferedIndexInput, TestMultiThreadTermVectors, TestConcurrentMergeScheduler, TestFilterDirectoryReader, TestDocumentsWriterStallControl, TestBinaryDocument, TestOperations, TestComplexExplanationsOfNonMatches, TestConsistentFieldNumbers, TestIndexWriterNRTIsCurrent, TestSentinelIntSet, TestStressDeletes, TestTermsEnum2, TestCodecHoldsOpenFiles, TestTermVectorsWriter, TestMultiCollector, TestNamedSPILoader, TestBooleanOr, TestStringMSBRadixSorter, TestIndexFileDeleter, TestSortedNumericSortField, TestMergedIterator, Test2BBinaryDocValues, TestStressNRT, TestBoostQuery, TestPayloadsOnVectors, TestDisjunctionMaxQuery, TestIndexWriterMaxDocs, TestIndexWriterDeleteByQuery, TestBytesStore, TestForUtil, TestSpanExplanationsOfNonMatches, TestCharsRef, TestIntsRef, TestLRUQueryCache, TestTieredMergePolicy, TestOrdinalMap, TestUpgradeIndexMergePolicy, TestNativeFSLockFactory, TestTerms, TestFrequencyTrackingRingBuffer, TestFieldValueQuery, TestDocumentWriter, TestTopDocsCollector, TestUTF32ToUTF8, TestSpanExplanations, TestSpanNearQuery, FiniteStringsIteratorTest, TestLongPostings, TestDocBoost, TestSloppyPhraseQuery, TestBooleanMinShouldMatch, TestDirectoryReader, TestSizeBoundedForceMerge, Test2BPostings, TestSameScoresWithThreads, TestSparseFixedBitDocIdSet, TestWildcard, TestDeletionPolicy, TestBasics, TestDuelingCodecs, TestSearchWithThreads, TestStressIndexing, TestRollingUpdates, TestSnapshotDeletionPolicy, TestIndexWriterUnicode, TestNRTReaderWithThreads, TestIndexWriterForceMerge, TestArrayUtil, TestLevenshteinAutomata, TestByteSlices, TestLockFactory, TestSegmentMerger, TestPhraseQuery, TestCollectionUtil, TestBagOfPositions, TestOmitNorms, TestFixedBitSet, TestRegexpRandom2, TestNRTCachingDirectory, TestSegmentReader, TestFastDecompressionMode, TestFieldCacheRewriteMethod, TestPagedBytes, TestPerFieldPostingsFormat2, TestMultiDocValues, TestSimpleSearchEquivalence, TestCustomSearcherSort, TestDocValuesIndexing, TestHugeRamFile, TestDeterminizeLexicon, TestIndexWriterOnDiskFull, TestDeterminism, TestBagOfPostings, TestCachingCollector, TestTotalHitCountCollector, TestRamUsageEstimator, TestNoMergePolicy, TestIndexWriterLockRelease, TestPhrasePrefixQuery, TestByteBlockPool, TestDemo, TestAssertions, TestRollback, TestNot, TestCloseableThreadLocal, TestReaderClosed, TestScoreCachingWrappingScorer, TestTermdocPerf, Test2BPostingsBytes, TestPositiveScoresOnlyCollector, TestIOUtils, Test2BTerms, TestNewestSegment, TestByteArrayDataInput, Test2BNumericDocValues, Test2BPagedBytes, TestCharArrayMap, TestCharArraySet, TestCharacterUtils, TestWordlistLoader, TestBytesRefAttImpl, TestGrowableByteArrayDataOutput, TestLucene50StoredFieldsFormatHighCompression, TestDemoParallelLeafReader]
   [junit4] Completed [387/439 (1!)] on J2 in 151.15s, 5 tests, 2 errors <<< FAILURES!

[...truncated 10772 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/init-core-data-001
   [junit4]   2> 2027294 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[A5006D8D59AE1922]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2027294 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[A5006D8D59AE1922]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 2027296 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2027296 INFO  (Thread-14035) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2027296 INFO  (Thread-14035) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2027396 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.ZkTestServer start zk server on port:56215
   [junit4]   2> 2027404 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2027405 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2027406 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2027407 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2027408 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2027409 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2027410 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2027410 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2027412 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2027413 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2027413 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2027489 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/control-001/cores/collection1
   [junit4]   2> 2027491 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 2027492 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7012a95e{/,null,AVAILABLE}
   [junit4]   2> 2027496 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.ServerConnector Started ServerConnector@278a6eb4{SSL,[ssl, http/1.1]}{127.0.0.1:45458}
   [junit4]   2> 2027496 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.Server Started @2029677ms
   [junit4]   2> 2027496 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/tempDir-001/control/data, hostContext=/, hostPort=45458, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/control-001/cores}
   [junit4]   2> 2027496 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.3.0
   [junit4]   2> 2027496 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2027496 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2027496 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-13T12:18:31.301Z
   [junit4]   2> 2027508 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2027508 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/control-001/solr.xml
   [junit4]   2> 2027513 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2027519 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56215/solr
   [junit4]   2> 2027530 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:45458_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45458_
   [junit4]   2> 2027531 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:45458_    ] o.a.s.c.Overseer Overseer (id=96754801777180677-127.0.0.1:45458_-n_0000000000) starting
   [junit4]   2> 2027543 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:45458_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45458_
   [junit4]   2> 2027549 INFO  (OverseerStateUpdate-96754801777180677-127.0.0.1:45458_-n_0000000000) [n:127.0.0.1:45458_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2027554 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:45458_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/control-001/cores
   [junit4]   2> 2027554 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:45458_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2027556 INFO  (OverseerStateUpdate-96754801777180677-127.0.0.1:45458_-n_0000000000) [n:127.0.0.1:45458_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 2028566 WARN  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2028567 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 2028586 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2028646 WARN  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2028647 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2028654 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 2028654 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/control-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/control-001/cores/collection1/data/]
   [junit4]   2> 2028654 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@414d1c34
   [junit4]   2> 2028655 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1be0a7ac
   [junit4]   2> 2028658 WARN  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 2028662 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2028663 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2028663 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2028663 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2028663 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@75b8fed
   [junit4]   2> 2028663 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@51455aac[collection1] main]
   [junit4]   2> 2028664 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2028664 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2028664 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2028665 INFO  (searcherExecutor-4487-thread-1-processing-n:127.0.0.1:45458_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@51455aac[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2028665 INFO  (coreLoadExecutor-4486-thread-1-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1548076829869342720
   [junit4]   2> 2028670 INFO  (coreZkRegister-4479-thread-1-processing-n:127.0.0.1:45458_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2028670 INFO  (coreZkRegister-4479-thread-1-processing-n:127.0.0.1:45458_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2028670 INFO  (coreZkRegister-4479-thread-1-processing-n:127.0.0.1:45458_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:45458/collection1/
   [junit4]   2> 2028670 INFO  (coreZkRegister-4479-thread-1-processing-n:127.0.0.1:45458_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2028671 INFO  (coreZkRegister-4479-thread-1-processing-n:127.0.0.1:45458_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:45458/collection1/ has no replicas
   [junit4]   2> 2028673 INFO  (coreZkRegister-4479-thread-1-processing-n:127.0.0.1:45458_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:45458/collection1/ shard1
   [junit4]   2> 2028824 INFO  (coreZkRegister-4479-thread-1-processing-n:127.0.0.1:45458_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:45458_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2029058 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2029058 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2029130 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-1-001/cores/collection1
   [junit4]   2> 2029130 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-1-001
   [junit4]   2> 2029132 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 2029133 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1e2a53c2{/,null,AVAILABLE}
   [junit4]   2> 2029134 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.ServerConnector Started ServerConnector@48a28973{SSL,[ssl, http/1.1]}{127.0.0.1:51439}
   [junit4]   2> 2029134 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.Server Started @2031315ms
   [junit4]   2> 2029134 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=51439, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-1-001/cores}
   [junit4]   2> 2029135 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.3.0
   [junit4]   2> 2029135 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2029135 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2029135 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-13T12:18:32.940Z
   [junit4]   2> 2029139 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2029139 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-1-001/solr.xml
   [junit4]   2> 2029143 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2029144 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56215/solr
   [junit4]   2> 2029151 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:51439_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2029152 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:51439_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51439_
   [junit4]   2> 2029154 INFO  (zkCallback-1969-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2029154 INFO  (zkCallback-1965-thread-2-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2029160 INFO  (zkCallback-1974-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2029176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:51439_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-1-001/cores
   [junit4]   2> 2029176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:51439_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2029178 INFO  (OverseerStateUpdate-96754801777180677-127.0.0.1:45458_-n_0000000000) [n:127.0.0.1:45458_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 2030187 WARN  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2030188 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 2030201 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2030270 WARN  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2030272 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2030280 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2030280 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-1-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 2030280 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@414d1c34
   [junit4]   2> 2030282 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@49e96323
   [junit4]   2> 2030284 WARN  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 2030291 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2030291 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2030291 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2030291 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2030293 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1697d355
   [junit4]   2> 2030293 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@49db865c[collection1] main]
   [junit4]   2> 2030294 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2030294 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2030294 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2030295 INFO  (searcherExecutor-4498-thread-1-processing-n:127.0.0.1:51439_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@49db865c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2030296 INFO  (coreLoadExecutor-4497-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1548076831579570176
   [junit4]   2> 2030301 INFO  (coreZkRegister-4492-thread-1-processing-n:127.0.0.1:51439_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2030301 INFO  (coreZkRegister-4492-thread-1-processing-n:127.0.0.1:51439_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2030301 INFO  (coreZkRegister-4492-thread-1-processing-n:127.0.0.1:51439_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:51439/collection1/
   [junit4]   2> 2030301 INFO  (coreZkRegister-4492-thread-1-processing-n:127.0.0.1:51439_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2030302 INFO  (coreZkRegister-4492-thread-1-processing-n:127.0.0.1:51439_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:51439/collection1/ has no replicas
   [junit4]   2> 2030304 INFO  (coreZkRegister-4492-thread-1-processing-n:127.0.0.1:51439_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:51439/collection1/ shard2
   [junit4]   2> 2030455 INFO  (coreZkRegister-4492-thread-1-processing-n:127.0.0.1:51439_ x:collection1 s:shard2 c:collection1 r:core_node1) [n:127.0.0.1:51439_ c:collection1 s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2030780 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-2-001/cores/collection1
   [junit4]   2> 2030784 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-2-001
   [junit4]   2> 2030796 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 2030797 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6ec1d568{/,null,AVAILABLE}
   [junit4]   2> 2030809 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.ServerConnector Started ServerConnector@3a9ea3{SSL,[ssl, http/1.1]}{127.0.0.1:48325}
   [junit4]   2> 2030809 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.e.j.s.Server Started @2032990ms
   [junit4]   2> 2030809 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=48325, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-2-001/cores}
   [junit4]   2> 2030810 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 6.3.0
   [junit4]   2> 2030810 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2030810 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2030810 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-13T12:18:34.615Z
   [junit4]   2> 2030815 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 2030815 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-2-001/solr.xml
   [junit4]   2> 2030820 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2030825 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56215/solr
   [junit4]   2> 2030861 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:48325_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2030863 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:48325_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48325_
   [junit4]   2> 2030864 INFO  (zkCallback-1969-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2030864 INFO  (zkCallback-1980-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2030865 INFO  (zkCallback-1974-thread-1-processing-n:127.0.0.1:51439_) [n:127.0.0.1:51439_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2030868 INFO  (zkCallback-1965-thread-4-processing-n:127.0.0.1:45458_) [n:127.0.0.1:45458_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2030884 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:48325_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-2-001/cores
   [junit4]   2> 2030884 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[A5006D8D59AE1922]) [n:127.0.0.1:48325_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 2030893 INFO  (OverseerStateUpdate-96754801777180677-127.0.0.1:45458_-n_0000000000) [n:127.0.0.1:45458_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 2031900 WARN  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1   x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2031901 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
   [junit4]   2> 2031911 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2031980 WARN  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2031981 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2031992 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2031992 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-2-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_A5006D8D59AE1922-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 2031992 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@414d1c34
   [junit4]   2> 2031995 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@34787a2d
   [junit4]   2> 2032001 WARN  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 2032011 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2032011 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2032011 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2032011 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2032022 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3ae87926
   [junit4]   2> 2032022 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4fc159dd[collection1] main]
   [junit4]   2> 2032022 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2032023 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2032027 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2032027 INFO  (coreLoadExecutor-4508-thread-1-processing-n:127.0.0.1:48325_) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1548076833394655232
   [junit4]   2> 2032029 INFO  (searcherExecutor-4509-thread-1-processing-n:127.0.0.1:48325_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4fc159dd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2032033 INFO  (coreZkRegister-4503-thread-1-processing-n:127.0.0.1:48325_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2032033 INFO  (coreZkRegister-4503-thread-1-processing-n:127.0.0.1:48325_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2032033 INFO  (coreZkRegister-4503-thread-1-processing-n:127.0.0.1:48325_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:48325_ c:collection1 s:shard1 r:core_node2 x:collection1] 

[...truncated too long message...]

cheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-6.3.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J1-20161013_114441_587.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J1-20161013_114441_587.suites -stdin
   [junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:773: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:709: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build.xml:252: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 254 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




Re: [JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 174 - Still Failing

Posted by Dawid Weiss <da...@gmail.com>.
No heapdump, but the log will tell you what happened:
https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/174/consoleText

At the end you see this:

   [junit4] ERROR: JVM J0 ended with an exception: Quit event not
received from the forked process? This may indicate JVM crash or
runner bugs.

And indeed, when you look at the logs, this is what it has to say:

   [junit4] <<< JVM J0: EOF ----
   [junit4] JVM J0: stderr was not empty, see:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/temp/junit4-J0-20161012_135246_796.syserr
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4]
   [junit4] Exception: java.lang.OutOfMemoryError thrown from the
UncaughtExceptionHandler in thread
"OldIndexDirectoryCleanupThreadForCore-awholynewstresscollection_collection1_2_shard3_replica2"
   [junit4] WARN: Unhandled exception in event serialization. ->
java.lang.OutOfMemoryError: GC overhead limit exceeded

These are the types or exceptions that are very difficult to guard
against -- GC overhead limit simply happens when the JVM is low on
memory and GC can't keep up with cleaning up. Here it happened on
event serialization back to the master process controlling JUnit
execution.

The runner has to have enough room to serialize events, otherwise the
results will be hard to predict or control.

Dawid

P.S. The entire log dump contains more stack dumps, but these seem to
be dumped from Solr -- perhaps there is an obscured reason for running
so low on memory there.

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 174 - Still Failing

Posted by Mikhail Khludnev <mk...@apache.org>.
Hello, dev!
Is there  a chance to get a heapdump from there?

On Wed, Oct 12, 2016 at 6:24 PM, Apache Jenkins Server <
jenkins@builds.apache.org> wrote:

> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/174/
>
> 2 tests failed.
> FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=24687,
> name=Thread-11778, state=RUNNABLE, group=TGRP-
> FullSolrCloudDistribCmdsTest]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=24687, name=Thread-11778,
> state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
> Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException:
> Timeout occured while waiting response from server at:
> http://127.0.0.1:51065/xvo/mi/collection1
>         at __randomizedtesting.SeedInfo.seed([C3DFCCDCCB98882]:0)
>         at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$
> 1IndexThread.run(FullSolrCloudDistribCmdsTest.java:644)
> Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout
> occured while waiting response from server at:
> http://127.0.0.1:51065/xvo/mi/collection1
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:604)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:262)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:251)
>         at org.apache.solr.client.solrj.SolrRequest.process(
> SolrRequest.java:149)
>         at org.apache.solr.client.solrj.SolrRequest.process(
> SolrRequest.java:166)
>         at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$
> 1IndexThread.run(FullSolrCloudDistribCmdsTest.java:642)
> Caused by: java.net.SocketTimeoutException: Read timed out
>         at java.net.SocketInputStream.socketRead0(Native Method)
>         at java.net.SocketInputStream.socketRead(SocketInputStream.
> java:116)
>         at java.net.SocketInputStream.read(SocketInputStream.java:170)
>         at java.net.SocketInputStream.read(SocketInputStream.java:141)
>         at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(
> AbstractSessionInputBuffer.java:160)
>         at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(
> SocketInputBuffer.java:84)
>         at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(
> AbstractSessionInputBuffer.java:273)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:140)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:498)
>         ... 5 more
>
>
> FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test
>
> Error Message:
> Timeout occured while waiting response from server at:
> https://127.0.0.1:35901
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: Timeout occured while
> waiting response from server at: https://127.0.0.1:35901
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:604)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:262)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:251)
>         at org.apache.solr.client.solrj.SolrClient.request(SolrClient.
> java:1219)
>         at org.apache.solr.cloud.CollectionsAPIDistributedZkTes
> t.makeRequest(CollectionsAPIDistributedZkTest.java:399)
>         at org.apache.solr.cloud.CollectionsAPIDistributedZkTes
> t.testErrorHandling(CollectionsAPIDistributedZkTest.java:437)
>         at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(
> CollectionsAPIDistributedZkTest.java:179)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:62)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:498)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1764)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:871)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(
> RandomizedRunner.java:907)
>         at com.carrotsearch.randomizedtesting.
> RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:57)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:49)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:45)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:47)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:367)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:809)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:880)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:816)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:57)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:45)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:41)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:40)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:53)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:47)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:54)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:367)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketTimeoutException: Read timed out
>         at java.net.SocketInputStream.socketRead0(Native Method)
>         at java.net.SocketInputStream.socketRead(SocketInputStream.
> java:116)
>         at java.net.SocketInputStream.read(SocketInputStream.java:170)
>         at java.net.SocketInputStream.read(SocketInputStream.java:141)
>         at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
>         at sun.security.ssl.InputRecord.read(InputRecord.java:503)
>         at sun.security.ssl.SSLSocketImpl.readRecord(
> SSLSocketImpl.java:973)
>         at sun.security.ssl.SSLSocketImpl.readDataRecord(
> SSLSocketImpl.java:930)
>         at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
>         at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(
> AbstractSessionInputBuffer.java:160)
>         at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(
> SocketInputBuffer.java:84)
>         at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(
> AbstractSessionInputBuffer.java:273)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:140)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:498)
>         ... 47 more
>
>
>
>
> Build Log:
> [...truncated 11659 lines...]
>    [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/
> temp/junit4-J0-20161012_135246_796.sysout
>    [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
>    [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4] Dumping heap to /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/heapdumps/java_pid6257.hprof
> ...
>    [junit4] Heap dump file created [656594589 bytes in 3.728 secs]
>    [junit4] <<< JVM J0: EOF ----
>
>    [junit4] JVM J0: stderr was not empty, see: /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/test/
> temp/junit4-J0-20161012_135246_796.syserr
>    [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
>    [junit4]
>    [junit4] Exception: java.lang.OutOfMemoryError thrown from the
> UncaughtExceptionHandler in thread "OldIndexDirectoryCleanupThreadForCore-
> awholynewstresscollection_collection1_2_shard3_replica2"
>    [junit4] WARN: Unhandled exception in event serialization. ->
> java.lang.OutOfMemoryError: GC overhead limit exceeded
>    [junit4] <<< JVM J0: EOF ----
>
> [...truncated 170 lines...]
>    [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
>    [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/init-core-data-001
>    [junit4]   2> 1721364 INFO  (SUITE-CollectionsAPIDistributedZkTes
> t-seed#[C3DFCCDCCB98882]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized
> ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=,
> ssl=NaN, value=NaN, clientAuth=NaN)
>    [junit4]   2> 1721364 INFO  (SUITE-CollectionsAPIDistributedZkTes
> t-seed#[C3DFCCDCCB98882]-worker) [    ] o.a.s.BaseDistributedSearchTestCase
> Setting hostContext system property: /
>    [junit4]   2> 1721366 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
>    [junit4]   2> 1721366 INFO  (Thread-11245) [    ] o.a.s.c.ZkTestServer
> client port:0.0.0.0/0.0.0.0:0
>    [junit4]   2> 1721366 INFO  (Thread-11245) [    ] o.a.s.c.ZkTestServer
> Starting server
>    [junit4]   2> 1721466 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.ZkTestServer start zk server on port:57186
>    [junit4]   2> 1721472 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0)
> [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
>    [junit4]   2> EndOfStreamException: Unable to read additional data from
> client sessionid 0x157b94503d50000, likely client has closed socket
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxn.doIO(
> NIOServerCnxn.java:228)
>    [junit4]   2>        at org.apache.zookeeper.server.
> NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>    [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> 1721477 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to
> /configs/conf1/solrconfig.xml
>    [junit4]   2> 1721479 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/schema.xml to
> /configs/conf1/schema.xml
>    [junit4]   2> 1721480 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
> to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 1721481 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/stopwords.txt to
> /configs/conf1/stopwords.txt
>    [junit4]   2> 1721482 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/protwords.txt to
> /configs/conf1/protwords.txt
>    [junit4]   2> 1721483 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/currency.xml to
> /configs/conf1/currency.xml
>    [junit4]   2> 1721484 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to
> /configs/conf1/enumsConfig.xml
>    [junit4]   2> 1721485 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
> to /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 1721486 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
> to /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 1721487 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to
> /configs/conf1/old_synonyms.txt
>    [junit4]   2> 1721488 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/src/test-files/solr/collection1/conf/synonyms.txt to
> /configs/conf1/synonyms.txt
>    [junit4]   2> 1721491 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0)
> [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
>    [junit4]   2> EndOfStreamException: Unable to read additional data from
> client sessionid 0x157b94503d50002, likely client has closed socket
>    [junit4]   2>        at org.apache.zookeeper.server.NIOServerCnxn.doIO(
> NIOServerCnxn.java:228)
>    [junit4]   2>        at org.apache.zookeeper.server.
> NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>    [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> 1721556 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-
> 001/cores/collection1
>    [junit4]   2> 1721558 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server jetty-9.3.8.v20160314
>    [junit4]   2> 1721559 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@
> 5f063735{/,null,AVAILABLE}
>    [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.ServerConnector Started ServerConnector@1d14519b{SSL,[ssl,
> http/1.1]}{127.0.0.1:33163}
>    [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server Started @1723802ms
>    [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties:
> {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/
> test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTes
> t_C3DFCCDCCB98882-001/tempDir-001/control/data, hostContext=/,
> hostPort=33163, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/
> test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTes
> t_C3DFCCDCCB98882-001/control-001/cores}
>    [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™
> version 6.3.0
>    [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode
> on port null
>    [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
>    [junit4]   2> 1721562 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time:
> 2016-10-12T14:21:30.648Z
>    [junit4]   2> 1721564 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found
> in ZooKeeper)
>    [junit4]   2> 1721564 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.SolrXmlConfig Loading container configuration from
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/solr.xml
>    [junit4]   2> 1721577 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client
> with params: socketTimeout=340000&connTimeout=45000&retry=true
>    [junit4]   2> 1721577 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
>    [junit4]   2> 1721592 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:33163_    ] o.a.s.c.OverseerElectionContext I am going to be
> the leader 127.0.0.1:33163_
>    [junit4]   2> 1721593 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:33163_    ] o.a.s.c.Overseer Overseer
> (id=96749623079927813-127.0.0.1:33163_-n_0000000000) starting
>    [junit4]   2> 1721596 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:33163_    ] o.a.s.c.ZkController Register node as live in
> ZooKeeper:/live_nodes/127.0.0.1:33163_
>    [junit4]   2> 1721597 INFO  (OverseerStateUpdate-
> 96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
>    [junit4]   2> 1721605 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:33163_    ] o.a.s.c.CorePropertiesLocator Found 1 core
> definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/cores
>    [junit4]   2> 1721605 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:33163_    ] o.a.s.c.CorePropertiesLocator Cores are:
> [collection1]
>    [junit4]   2> 1721606 INFO  (OverseerStateUpdate-
> 96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ]
> o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
>    [junit4]   2> 1721617 WARN  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_
> c:control_collection   x:collection1] o.a.s.c.Config Beginning with Solr
> 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
>    [junit4]   2> 1721618 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_
> c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene
> MatchVersion: 6.3.0
>    [junit4]   2> 1721627 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_
> c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1]
> Schema name=test
>    [junit4]   2> 1721692 WARN  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_
> c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1]
> default search field in schema is text. WARNING: Deprecated, please use
> 'df' on request instead.
>    [junit4]   2> 1721693 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_
> c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema
> test/1.0 with uniqueid field id
>    [junit4]   2> 1721700 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_
> c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating
> SolrCore 'collection1' using configuration from collection
> control_collection
>    [junit4]   2> 1721701 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ]
> Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-001/cores/collection1],
> dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/control-
> 001/cores/collection1/data/]
>    [junit4]   2> 1721701 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring
> is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.
> JmxMBeanServer@365bd187
>    [junit4]   2> 1721701 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy:
> [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34,
> maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=0.0]
>    [junit4]   2> 1721704 WARN  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID
> paramSet a in requestHandler {type = requestHandler,name = /dump,class =
> DumpRequestHandler,args = {defaults={a=A,b=B}}}
>    [junit4]   2> 1721709 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog
> implementation: org.apache.solr.update.UpdateLog
>    [junit4]   2> 1721709 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing
> UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100
> maxNumLogsToKeep=10 numVersionBuckets=65536
>    [junit4]   2> 1721709 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit:
> disabled
>    [junit4]   2> 1721709 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit:
> disabled
>    [junit4]   2> 1721710 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14,
> maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625,
> forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
>    [junit4]   2> 1721710 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening
> [Searcher@28f83cc2[collection1] main]
>    [junit4]   2> 1721710 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage
> Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 1721711 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded
> null at path _rest_managed.json using ZooKeeperStorageIO:path=/
> configs/conf1
>    [junit4]   2> 1721711 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits
> will be reserved for  10000
>    [junit4]   2> 1721711 INFO  (searcherExecutor-5526-thread-
> 1-processing-n:127.0.0.1:33163_ x:collection1 s:shard1
> c:control_collection r:core_node1) [n:127.0.0.1:33163_
> c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore
> [collection1] Registered new searcher Searcher@28f83cc2[collection1]
> main{ExitableDirectoryReader(UninvertingDirectoryReader())}
>    [junit4]   2> 1721711 INFO  (coreLoadExecutor-5525-thread-
> 1-processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_ c:control_collection
> s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max
> version in index or recent updates, using new clock 1547993969639555072
>    [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-
> processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection
> r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1
> r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough
> replicas found to continue.
>    [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-
> processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection
> r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1
> r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be
> the new leader - try and sync
>    [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-
> processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection
> r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1
> r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to
> https://127.0.0.1:33163/collection1/
>    [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-
> processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection
> r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1
> r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync
> replicas to me
>    [junit4]   2> 1721716 INFO  (coreZkRegister-5518-thread-1-
> processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection
> r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1
> r:core_node1 x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:33163/
> collection1/ has no replicas
>    [junit4]   2> 1721719 INFO  (coreZkRegister-5518-thread-1-
> processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection
> r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1
> r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the
> new leader: https://127.0.0.1:33163/collection1/ shard1
>    [junit4]   2> 1721870 INFO  (coreZkRegister-5518-thread-1-
> processing-n:127.0.0.1:33163_ x:collection1 s:shard1 c:control_collection
> r:core_node1) [n:127.0.0.1:33163_ c:control_collection s:shard1
> r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no
> recovery necessary
>    [junit4]   2> 1722109 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) ->
> (1)
>    [junit4]   2> 1722110 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause
> connection loss:false
>    [junit4]   2> 1722175 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-
> 001/cores/collection1
>    [junit4]   2> 1722176 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001
>    [junit4]   2> 1722177 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server jetty-9.3.8.v20160314
>    [junit4]   2> 1722178 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@
> 78549ff9{/,null,AVAILABLE}
>    [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.ServerConnector Started ServerConnector@78092f07{SSL,[ssl,
> http/1.1]}{127.0.0.1:35901}
>    [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server Started @1724420ms
>    [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties:
> {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/
> test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTes
> t_C3DFCCDCCB98882-001/tempDir-001/jetty1, solrconfig=solrconfig.xml,
> hostContext=/, hostPort=35901, coreRootDirectory=/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/cores}
>    [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™
> version 6.3.0
>    [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode
> on port null
>    [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
>    [junit4]   2> 1722180 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time:
> 2016-10-12T14:21:31.266Z
>    [junit4]   2> 1722182 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found
> in ZooKeeper)
>    [junit4]   2> 1722182 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.SolrXmlConfig Loading container configuration from
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/solr.xml
>    [junit4]   2> 1722186 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client
> with params: socketTimeout=340000&connTimeout=45000&retry=true
>    [junit4]   2> 1722187 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
>    [junit4]   2> 1722192 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:35901_    ] o.a.s.c.c.ZkStateReader Updated live nodes from
> ZooKeeper... (0) -> (1)
>    [junit4]   2> 1722195 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:35901_    ] o.a.s.c.ZkController Register node as live in
> ZooKeeper:/live_nodes/127.0.0.1:35901_
>    [junit4]   2> 1722196 INFO  (zkCallback-2718-thread-1-
> processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
>    [junit4]   2> 1722196 INFO  (zkCallback-2727-thread-1-
> processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
>    [junit4]   2> 1722196 INFO  (zkCallback-2722-thread-1) [    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
>    [junit4]   2> 1722207 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:35901_    ] o.a.s.c.CorePropertiesLocator Found 1 core
> definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/cores
>    [junit4]   2> 1722207 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:35901_    ] o.a.s.c.CorePropertiesLocator Cores are:
> [collection1]
>    [junit4]   2> 1722208 INFO  (OverseerStateUpdate-
> 96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ]
> o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
>    [junit4]   2> 1723221 WARN  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
>  x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is
> deprecated, use <mergePolicyFactory> instead.
>    [junit4]   2> 1723222 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
>  x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
>    [junit4]   2> 1723232 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
>    [junit4]   2> 1723302 WARN  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema [collection1] default search field in
> schema is text. WARNING: Deprecated, please use 'df' on request instead.
>    [junit4]   2> 1723304 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid
> field id
>    [junit4]   2> 1723313 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
>  x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using
> configuration from collection collection1
>    [junit4]   2> 1723313 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ]
> Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-001/cores/collection1],
> dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-1-
> 001/cores/collection1/data/]
>    [junit4]   2> 1723313 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring
> is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.
> JmxMBeanServer@365bd187
>    [junit4]   2> 1723314 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy:
> [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34,
> maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=0.0]
>    [junit4]   2> 1723317 WARN  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID
> paramSet a in requestHandler {type = requestHandler,name = /dump,class =
> DumpRequestHandler,args = {defaults={a=A,b=B}}}
>    [junit4]   2> 1723322 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog
> implementation: org.apache.solr.update.UpdateLog
>    [junit4]   2> 1723322 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing
> UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100
> maxNumLogsToKeep=10 numVersionBuckets=65536
>    [junit4]   2> 1723323 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit:
> disabled
>    [junit4]   2> 1723323 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit:
> disabled
>    [junit4]   2> 1723324 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14,
> maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625,
> forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
>    [junit4]   2> 1723324 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening
> [Searcher@61147bc7[collection1] main]
>    [junit4]   2> 1723325 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage
> Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 1723325 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded
> null at path _rest_managed.json using ZooKeeperStorageIO:path=/
> configs/conf1
>    [junit4]   2> 1723325 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits
> will be reserved for  10000
>    [junit4]   2> 1723326 INFO  (searcherExecutor-5537-thread-
> 1-processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1
> r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1
> x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher
> Searcher@61147bc7[collection1] main{ExitableDirectoryReader(
> UninvertingDirectoryReader())}
>    [junit4]   2> 1723326 INFO  (coreLoadExecutor-5536-thread-
> 1-processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_ c:collection1
> s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max
> version in index or recent updates, using new clock 1547993971333005312
>    [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-
> processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1
> r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1
> x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found
> to continue.
>    [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-
> processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1
> r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1
> x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader
> - try and sync
>    [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-
> processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1
> r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1
> x:collection1] o.a.s.c.SyncStrategy Sync replicas to
> https://127.0.0.1:35901/collection1/
>    [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-
> processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1
> r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1
> x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
>    [junit4]   2> 1723331 INFO  (coreZkRegister-5531-thread-1-
> processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1
> r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1
> x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:35901/collection1/
> has no replicas
>    [junit4]   2> 1723334 INFO  (coreZkRegister-5531-thread-1-
> processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1
> r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1
> x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader:
> https://127.0.0.1:35901/collection1/ shard2
>    [junit4]   2> 1723484 INFO  (coreZkRegister-5531-thread-1-
> processing-n:127.0.0.1:35901_ x:collection1 s:shard2 c:collection1
> r:core_node1) [n:127.0.0.1:35901_ c:collection1 s:shard2 r:core_node1
> x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
>    [junit4]   2> 1723777 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-
> 001/cores/collection1
>    [junit4]   2> 1723777 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001
>    [junit4]   2> 1723779 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server jetty-9.3.8.v20160314
>    [junit4]   2> 1723780 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@
> 70afaa33{/,null,AVAILABLE}
>    [junit4]   2> 1723781 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.ServerConnector Started ServerConnector@518924{SSL,[ssl,
> http/1.1]}{127.0.0.1:59763}
>    [junit4]   2> 1723781 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server Started @1726022ms
>    [junit4]   2> 1723781 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties:
> {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/
> test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTes
> t_C3DFCCDCCB98882-001/tempDir-001/jetty2, solrconfig=solrconfig.xml,
> hostContext=/, hostPort=59763, coreRootDirectory=/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/build/solr-core/test/J2/../../../../../../../.
> ./../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/cores}
>    [junit4]   2> 1723782 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™
> version 6.3.0
>    [junit4]   2> 1723782 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode
> on port null
>    [junit4]   2> 1723782 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
>    [junit4]   2> 1723782 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time:
> 2016-10-12T14:21:32.868Z
>    [junit4]   2> 1723786 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found
> in ZooKeeper)
>    [junit4]   2> 1723786 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.SolrXmlConfig Loading container configuration from
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/solr.xml
>    [junit4]   2> 1723791 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client
> with params: socketTimeout=340000&connTimeout=45000&retry=true
>    [junit4]   2> 1723792 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
>    [junit4]   2> 1723800 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:59763_    ] o.a.s.c.c.ZkStateReader Updated live nodes from
> ZooKeeper... (0) -> (2)
>    [junit4]   2> 1723802 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:59763_    ] o.a.s.c.ZkController Register node as live in
> ZooKeeper:/live_nodes/127.0.0.1:59763_
>    [junit4]   2> 1723803 INFO  (zkCallback-2722-thread-1) [    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>    [junit4]   2> 1723803 INFO  (zkCallback-2727-thread-1-
> processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>    [junit4]   2> 1723803 INFO  (zkCallback-2718-thread-1-
> processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>    [junit4]   2> 1723805 INFO  (zkCallback-2733-thread-1-
> processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
>    [junit4]   2> 1723810 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:59763_    ] o.a.s.c.CorePropertiesLocator Found 1 core
> definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/.
> ./../../../../../../../../../../x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/cores
>    [junit4]   2> 1723810 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:59763_    ] o.a.s.c.CorePropertiesLocator Cores are:
> [collection1]
>    [junit4]   2> 1723811 INFO  (OverseerStateUpdate-
> 96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ]
> o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
>    [junit4]   2> 1724819 WARN  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
>  x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is
> deprecated, use <mergePolicyFactory> instead.
>    [junit4]   2> 1724820 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
>  x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
>    [junit4]   2> 1724830 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
>    [junit4]   2> 1724902 WARN  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema [collection1] default search field in
> schema is text. WARNING: Deprecated, please use 'df' on request instead.
>    [junit4]   2> 1724904 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid
> field id
>    [junit4]   2> 1724913 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
>  x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using
> configuration from collection collection1
>    [junit4]   2> 1724913 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [[collection1] ]
> Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-001/cores/collection1],
> dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/.
> ./../../../../../../../../../../x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-2-
> 001/cores/collection1/data/]
>    [junit4]   2> 1724913 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring
> is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.
> JmxMBeanServer@365bd187
>    [junit4]   2> 1724914 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy:
> [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34,
> maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=0.0]
>    [junit4]   2> 1724917 WARN  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.c.RequestHandlers INVALID
> paramSet a in requestHandler {type = requestHandler,name = /dump,class =
> DumpRequestHandler,args = {defaults={a=A,b=B}}}
>    [junit4]   2> 1724923 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog
> implementation: org.apache.solr.update.UpdateLog
>    [junit4]   2> 1724923 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Initializing
> UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100
> maxNumLogsToKeep=10 numVersionBuckets=65536
>    [junit4]   2> 1724923 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit:
> disabled
>    [junit4]   2> 1724923 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit:
> disabled
>    [junit4]   2> 1724924 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14,
> maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625,
> forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
>    [junit4]   2> 1724924 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.s.SolrIndexSearcher Opening
> [Searcher@5a83c8ea[collection1] main]
>    [junit4]   2> 1724925 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.r.ManagedResourceStorage
> Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 1724925 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.r.ManagedResourceStorage Loaded
> null at path _rest_managed.json using ZooKeeperStorageIO:path=/
> configs/conf1
>    [junit4]   2> 1724925 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.h.ReplicationHandler Commits
> will be reserved for  10000
>    [junit4]   2> 1724926 INFO  (searcherExecutor-5548-thread-
> 1-processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1
> r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2
> x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher
> Searcher@5a83c8ea[collection1] main{ExitableDirectoryReader(
> UninvertingDirectoryReader())}
>    [junit4]   2> 1724926 INFO  (coreLoadExecutor-5547-thread-
> 1-processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_ c:collection1
> s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Could not find max
> version in index or recent updates, using new clock 1547993973010726912
>    [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-
> processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1
> r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2
> x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found
> to continue.
>    [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-
> processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1
> r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2
> x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader
> - try and sync
>    [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-
> processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1
> r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2
> x:collection1] o.a.s.c.SyncStrategy Sync replicas to
> https://127.0.0.1:59763/collection1/
>    [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-
> processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1
> r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2
> x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
>    [junit4]   2> 1724932 INFO  (coreZkRegister-5542-thread-1-
> processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1
> r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2
> x:collection1] o.a.s.c.SyncStrategy https://127.0.0.1:59763/collection1/
> has no replicas
>    [junit4]   2> 1724934 INFO  (coreZkRegister-5542-thread-1-
> processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1
> r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2
> x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader:
> https://127.0.0.1:59763/collection1/ shard1
>    [junit4]   2> 1725085 INFO  (coreZkRegister-5542-thread-1-
> processing-n:127.0.0.1:59763_ x:collection1 s:shard1 c:collection1
> r:core_node2) [n:127.0.0.1:59763_ c:collection1 s:shard1 r:core_node2
> x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
>    [junit4]   2> 1725376 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-
> 001/cores/collection1
>    [junit4]   2> 1725376 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001
>    [junit4]   2> 1725377 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server jetty-9.3.8.v20160314
>    [junit4]   2> 1725379 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@
> 60026e9{/,null,AVAILABLE}
>    [junit4]   2> 1725381 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.ServerConnector Started ServerConnector@7c6e0e6f{SSL,[ssl,
> http/1.1]}{127.0.0.1:40922}
>    [junit4]   2> 1725381 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server Started @1727622ms
>    [junit4]   2> 1725381 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties:
> {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/
> test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTes
> t_C3DFCCDCCB98882-001/tempDir-001/jetty3, solrconfig=solrconfig.xml,
> hostContext=/, hostPort=40922, coreRootDirectory=/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/cores}
>    [junit4]   2> 1725382 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™
> version 6.3.0
>    [junit4]   2> 1725382 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode
> on port null
>    [junit4]   2> 1725382 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
>    [junit4]   2> 1725382 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time:
> 2016-10-12T14:21:34.468Z
>    [junit4]   2> 1725385 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found
> in ZooKeeper)
>    [junit4]   2> 1725385 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.SolrXmlConfig Loading container configuration from
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/solr.xml
>    [junit4]   2> 1725389 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client
> with params: socketTimeout=340000&connTimeout=45000&retry=true
>    [junit4]   2> 1725390 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
>    [junit4]   2> 1725398 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:40922_    ] o.a.s.c.c.ZkStateReader Updated live nodes from
> ZooKeeper... (0) -> (3)
>    [junit4]   2> 1725401 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:40922_    ] o.a.s.c.ZkController Register node as live in
> ZooKeeper:/live_nodes/127.0.0.1:40922_
>    [junit4]   2> 1725402 INFO  (zkCallback-2733-thread-1-
> processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
>    [junit4]   2> 1725402 INFO  (zkCallback-2722-thread-1) [    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
>    [junit4]   2> 1725402 INFO  (zkCallback-2739-thread-1-
> processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
>    [junit4]   2> 1725402 INFO  (zkCallback-2727-thread-1-
> processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
>    [junit4]   2> 1725402 INFO  (zkCallback-2718-thread-2-
> processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
>    [junit4]   2> 1725414 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:40922_    ] o.a.s.c.CorePropertiesLocator Found 1 core
> definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/cores
>    [junit4]   2> 1725414 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:40922_    ] o.a.s.c.CorePropertiesLocator Cores are:
> [collection1]
>    [junit4]   2> 1725415 INFO  (OverseerStateUpdate-
> 96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ]
> o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
>    [junit4]   2> 1726422 WARN  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
>  x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is
> deprecated, use <mergePolicyFactory> instead.
>    [junit4]   2> 1726423 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
>  x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
>    [junit4]   2> 1726433 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
>    [junit4]   2> 1726500 WARN  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema [collection1] default search field in
> schema is text. WARNING: Deprecated, please use 'df' on request instead.
>    [junit4]   2> 1726502 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid
> field id
>    [junit4]   2> 1726510 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
>  x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using
> configuration from collection collection1
>    [junit4]   2> 1726510 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.c.SolrCore [[collection1] ]
> Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-001/cores/collection1],
> dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-3-
> 001/cores/collection1/data/]
>    [junit4]   2> 1726510 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring
> is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.
> JmxMBeanServer@365bd187
>    [junit4]   2> 1726511 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy:
> [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34,
> maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=0.0]
>    [junit4]   2> 1726514 WARN  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.c.RequestHandlers INVALID
> paramSet a in requestHandler {type = requestHandler,name = /dump,class =
> DumpRequestHandler,args = {defaults={a=A,b=B}}}
>    [junit4]   2> 1726521 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog
> implementation: org.apache.solr.update.UpdateLog
>    [junit4]   2> 1726521 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Initializing
> UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100
> maxNumLogsToKeep=10 numVersionBuckets=65536
>    [junit4]   2> 1726521 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit:
> disabled
>    [junit4]   2> 1726521 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit:
> disabled
>    [junit4]   2> 1726522 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14,
> maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625,
> forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
>    [junit4]   2> 1726522 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.s.SolrIndexSearcher Opening
> [Searcher@2c3fcd7e[collection1] main]
>    [junit4]   2> 1726523 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.r.ManagedResourceStorage
> Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 1726523 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.r.ManagedResourceStorage Loaded
> null at path _rest_managed.json using ZooKeeperStorageIO:path=/
> configs/conf1
>    [junit4]   2> 1726523 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.h.ReplicationHandler Commits
> will be reserved for  10000
>    [junit4]   2> 1726524 INFO  (searcherExecutor-5559-thread-
> 1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher
> Searcher@2c3fcd7e[collection1] main{ExitableDirectoryReader(
> UninvertingDirectoryReader())}
>    [junit4]   2> 1726524 INFO  (coreLoadExecutor-5558-thread-
> 1-processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_ c:collection1
> s:shard2 r:core_node3 x:collection1] o.a.s.u.UpdateLog Could not find max
> version in index or recent updates, using new clock 1547993974686351360
>    [junit4]   2> 1726528 INFO  (coreZkRegister-5553-thread-1-
> processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
>    [junit4]   2> 1726528 INFO  (updateExecutor-2736-thread-1-
> processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
>    [junit4]   2> 1726528 INFO  (recoveryExecutor-2737-thread-
> 1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process.
> recoveringAfterStartup=true
>    [junit4]   2> 1726529 INFO  (recoveryExecutor-2737-thread-
> 1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
>    [junit4]   2> 1726529 INFO  (recoveryExecutor-2737-thread-
> 1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates.
> core=[collection1]
>    [junit4]   2> 1726529 INFO  (recoveryExecutor-2737-thread-
> 1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.u.UpdateLog Starting to buffer updates.
> FSUpdateLog{state=ACTIVE, tlog=null}
>    [junit4]   2> 1726529 INFO  (recoveryExecutor-2737-thread-
> 1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core
> [collection1] as recovering, leader is [https://127.0.0.1:35901/
> collection1/] and I am [https://127.0.0.1:40922/collection1/]
>    [junit4]   2> 1726531 INFO  (recoveryExecutor-2737-thread-
> 1-processing-n:127.0.0.1:40922_ x:collection1 s:shard2 c:collection1
> r:core_node3) [n:127.0.0.1:40922_ c:collection1 s:shard2 r:core_node3
> x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [
> https://127.0.0.1:35901]; [WaitForState: action=PREPRECOVERY&core=
> collection1&nodeName=127.0.0.1:40922_&coreNodeName=core_
> node3&state=recovering&checkLive=true&onlyIfLeader=
> true&onlyIfLeaderActive=true]
>    [junit4]   2> 1726588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_
>   ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node3,
> state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive:
> true
>    [junit4]   2> 1726588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_
>   ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see
> collection1 (shard2 of collection1) have state: recovering
>    [junit4]   2> 1726588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_
>   ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering):
> collection=collection1, shard=shard2, thisCore=collection1,
> leaderDoesNotNeedRecovery=false, isLeader? true, live=true,
> checkLive=true, currentState=down, localState=active,
> nodeName=127.0.0.1:40922_, coreNodeName=core_node3,
> onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"
> collection1","base_url":"https://127.0.0.1:40922","node_
> name":"127.0.0.1:40922_","state":"down"}
>    [junit4]   2> 1726979 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-
> 001/cores/collection1
>    [junit4]   2> 1726980 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001
>    [junit4]   2> 1726981 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server jetty-9.3.8.v20160314
>    [junit4]   2> 1726983 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@
> 2d232e59{/,null,AVAILABLE}
>    [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.ServerConnector Started ServerConnector@248524fa{SSL,[ssl,
> http/1.1]}{127.0.0.1:40836}
>    [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.e.j.s.Server Started @1729225ms
>    [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties:
> {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/build/solr-core/
> test/J2/temp/solr.cloud.CollectionsAPIDistributedZkTes
> t_C3DFCCDCCB98882-001/tempDir-001/jetty4, solrconfig=solrconfig.xml,
> hostContext=/, hostPort=40836, coreRootDirectory=/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/build/solr-core/test/J2/../../../../../../../.
> ./../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/cores}
>    [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™
> version 6.3.0
>    [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode
> on port null
>    [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
>    [junit4]   2> 1726984 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time:
> 2016-10-12T14:21:36.070Z
>    [junit4]   2> 1726987 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found
> in ZooKeeper)
>    [junit4]   2> 1726987 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.SolrXmlConfig Loading container configuration from
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/solr.xml
>    [junit4]   2> 1726994 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client
> with params: socketTimeout=340000&connTimeout=45000&retry=true
>    [junit4]   2> 1726995 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57186/solr
>    [junit4]   2> 1727001 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:40836_    ] o.a.s.c.c.ZkStateReader Updated live nodes from
> ZooKeeper... (0) -> (4)
>    [junit4]   2> 1727004 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:40836_    ] o.a.s.c.ZkController Register node as live in
> ZooKeeper:/live_nodes/127.0.0.1:40836_
>    [junit4]   2> 1727005 INFO  (zkCallback-2727-thread-1-
> processing-n:127.0.0.1:35901_) [n:127.0.0.1:35901_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
>    [junit4]   2> 1727005 INFO  (zkCallback-2733-thread-1-
> processing-n:127.0.0.1:59763_) [n:127.0.0.1:59763_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
>    [junit4]   2> 1727005 INFO  (zkCallback-2718-thread-3-
> processing-n:127.0.0.1:33163_) [n:127.0.0.1:33163_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
>    [junit4]   2> 1727005 INFO  (zkCallback-2739-thread-1-
> processing-n:127.0.0.1:40922_) [n:127.0.0.1:40922_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
>    [junit4]   2> 1727005 INFO  (zkCallback-2746-thread-1-
> processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
>    [junit4]   2> 1727005 INFO  (zkCallback-2722-thread-1) [    ]
> o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
>    [junit4]   2> 1727014 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:40836_    ] o.a.s.c.CorePropertiesLocator Found 1 core
> definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/.
> ./../../../../../../../../../../x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/cores
>    [junit4]   2> 1727014 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[C3DFCCDCCB98882])
> [n:127.0.0.1:40836_    ] o.a.s.c.CorePropertiesLocator Cores are:
> [collection1]
>    [junit4]   2> 1727016 INFO  (OverseerStateUpdate-
> 96749623079927813-127.0.0.1:33163_-n_0000000000) [n:127.0.0.1:33163_    ]
> o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
>    [junit4]   2> 1727588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_
>   ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering):
> collection=collection1, shard=shard2, thisCore=collection1,
> leaderDoesNotNeedRecovery=false, isLeader? true, live=true,
> checkLive=true, currentState=recovering, localState=active,
> nodeName=127.0.0.1:40922_, coreNodeName=core_node3,
> onlyIfActiveCheckResult=false, nodeProps: core_node3:{"core":"
> collection1","base_url":"https://127.0.0.1:40922","node_
> name":"127.0.0.1:40922_","state":"recovering"}
>    [junit4]   2> 1727588 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_
>   ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node3, state:
> recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
>    [junit4]   2> 1727589 INFO  (qtp2138804036-22184) [n:127.0.0.1:35901_
>   ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores
> params={nodeName=127.0.0.1:40922_&onlyIfLeaderActive=
> true&core=collection1&coreNodeName=core_node3&
> action=PREPRECOVERY&checkLive=true&state=recovering&
> onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1000
>    [junit4]   2> 1728024 WARN  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
>  x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is
> deprecated, use <mergePolicyFactory> instead.
>    [junit4]   2> 1728025 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
>  x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.3.0
>    [junit4]   2> 1728047 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
>    [junit4]   2> 1728107 WARN  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema [collection1] default search field in
> schema is text. WARNING: Deprecated, please use 'df' on request instead.
>    [junit4]   2> 1728109 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
>  x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid
> field id
>    [junit4]   2> 1728116 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
>  x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using
> configuration from collection collection1
>    [junit4]   2> 1728117 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.c.SolrCore [[collection1] ]
> Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-001/cores/collection1],
> dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/J2/.
> ./../../../../../../../../../../x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/J2/temp/solr.cloud.
> CollectionsAPIDistributedZkTest_C3DFCCDCCB98882-001/shard-4-
> 001/cores/collection1/data/]
>    [junit4]   2> 1728117 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring
> is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.
> JmxMBeanServer@365bd187
>    [junit4]   2> 1728117 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy:
> [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34,
> maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=0.0]
>    [junit4]   2> 1728121 WARN  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.c.RequestHandlers INVALID
> paramSet a in requestHandler {type = requestHandler,name = /dump,class =
> DumpRequestHandler,args = {defaults={a=A,b=B}}}
>    [junit4]   2> 1728127 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog
> implementation: org.apache.solr.update.UpdateLog
>    [junit4]   2> 1728127 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateLog Initializing
> UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100
> maxNumLogsToKeep=10 numVersionBuckets=65536
>    [junit4]   2> 1728135 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit:
> disabled
>    [junit4]   2> 1728135 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit:
> disabled
>    [junit4]   2> 1728136 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.u.RandomMergePolicy
> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy:
> [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=14,
> maxMergedSegmentMB=30.8408203125, floorSegmentMB=1.244140625,
> forceMergeDeletesPctAllowed=29.926466904599625, segmentsPerTier=45.0,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
>    [junit4]   2> 1728136 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.s.SolrIndexSearcher Opening
> [Searcher@40d4464b[collection1] main]
>    [junit4]   2> 1728137 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.r.ManagedResourceStorage
> Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 1728137 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.r.ManagedResourceStorage Loaded
> null at path _rest_managed.json using ZooKeeperStorageIO:path=/
> configs/conf1
>    [junit4]   2> 1728137 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.h.ReplicationHandler Commits
> will be reserved for  10000
>    [junit4]   2> 1728137 INFO  (searcherExecutor-5570-thread-
> 1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1
> r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4
> x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher
> Searcher@40d4464b[collection1] main{ExitableDirectoryReader(
> UninvertingDirectoryReader())}
>    [junit4]   2> 1728138 INFO  (coreLoadExecutor-5569-thread-
> 1-processing-n:127.0.0.1:40836_) [n:127.0.0.1:40836_ c:collection1
> s:shard1 r:core_node4 x:collection1] o.a.s.u.UpdateLog Could not find max
> version in index or recent updates, using new clock 1547993976378753024
>    [junit4]   2> 1728141 INFO  (coreZkRegister-5564-thread-1-
> processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1
> r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4
> x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
>    [junit4]   2> 1728142 INFO  (updateExecutor-2743-thread-1-
> processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1
> r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4
> x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
>    [junit4]   2> 1728142 INFO  (recoveryExecutor-2744-thread-
> 1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1
> r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4
> x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process.
> recoveringAfterStartup=true
>    [junit4]   2> 1728142 INFO  (recoveryExecutor-2744-thread-
> 1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1
> r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:core_node4
> x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
>    [junit4]   2> 1728142 INFO  (recoveryExecutor-2744-thread-
> 1-processing-n:127.0.0.1:40836_ x:collection1 s:shard1 c:collection1
> r:core_node4) [n:127.0.0.1:40836_ c:collection1 s:shard1 r:co
>
> [...truncated too long message...]
>
> cheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-
> slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-
> M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-
> interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-
> slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.
> jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-
> interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-
> slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-interceptors-journal-2.0.0-
> M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-
> interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/core/test-lib/apacheds-interceptors-
> operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-
> interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-
> slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.
> jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-
> interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-
> slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-
> M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-
> jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/
> x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/
> x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.
> jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/apacheds-
> protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/
> x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/
> x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/
> jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/core/test-lib/commons-collections-3.2.2.jar:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/
> jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/
> jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/hadoop-
> minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/
> htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/
> jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/jersey-
> core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/
> jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/jetty-6.1.
> 26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/jetty-util-
> 6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/
> mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/core/test-lib/mockito-
> core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/core/test-lib/
> netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-
> slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/
> jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/
> checkout/lucene/build/analysis/icu/lucene-analyzers-
> icu-6.3.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/contrib/solr-
> analysis-extras/classes/java:/x1/jenkins/jenkins-slave/
> workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/build/queryparser/
> classes/test:/x1/jenkins/jenkins-slave/workspace/
> Lucene-Solr-NightlyTests-6.x/checkout/solr/contrib/
> analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/
> apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/
> lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-
> 1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-
> 1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/
> apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/
> tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/
> tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/
> jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/
> home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.
> jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-
> antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-
> jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-
> apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/
> lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-
> ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-
> 1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-
> ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/
> apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/
> tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/
> jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.
> jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-
> apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-
> 1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-
> 1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/
> apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/
> asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.
> ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar
> com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/
> solr/build/solr-core/test/temp/junit4-J0-20161012_135246_796.events
> @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-
> NightlyTests-6.x/checkout/solr/build/solr-core/test/
> temp/junit4-J0-20161012_135246_796.suites -stdin
>    [junit4] ERROR: JVM J0 ended with an exception: Quit event not received
> from the forked process? This may indicate JVM crash or runner bugs.
>    [junit4]     at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(
> JUnit4.java:1501)
>    [junit4]     at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(
> JUnit4.java:121)
>    [junit4]     at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.
> java:955)
>    [junit4]     at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.
> java:952)
>    [junit4]     at java.util.concurrent.FutureTask.run(FutureTask.
> java:266)
>    [junit4]     at java.util.concurrent.ThreadPoolExecutor.runWorker(
> ThreadPoolExecutor.java:1142)
>    [junit4]     at java.util.concurrent.ThreadPoolExecutor$Worker.run(
> ThreadPoolExecutor.java:617)
>    [junit4]     at java.lang.Thread.run(Thread.java:745)
>
> BUILD FAILED
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:773:
> The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:709:
> The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/build.xml:59:
> The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/build.xml:252:
> The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/solr/common-build.xml:536:
> The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:1443:
> The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/checkout/lucene/common-build.xml:984:
> At least one slave process threw an exception, first: Quit event not
> received from the forked process? This may indicate JVM crash or runner
> bugs.
>
> Total time: 229 minutes 46 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
>
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>



-- 
Sincerely yours
Mikhail Khludnev