You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2016/05/05 18:04:29 UTC
[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 57 - Still Failing
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/57/
4 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test
Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:58336
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:58336
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:601)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:399)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:457)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:179)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:170)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:495)
... 47 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest: 1) Thread[id=7672, name=searcherExecutor-2383-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest:
1) Thread[id=7672, name=searcherExecutor-2383-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([E363401D1F6FC7BD]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=7672, name=searcherExecutor-2383-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=7672, name=searcherExecutor-2383-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at __randomizedtesting.SeedInfo.seed([E363401D1F6FC7BD]:0)
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test
Error Message:
Captured an uncaught exception in thread: Thread[id=115839, name=collection0, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=115839, name=collection0, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59048: Could not find collection : awholynewstresscollection_collection0_2
at __randomizedtesting.SeedInfo.seed([E363401D1F6FC7BD]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:590)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:404)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:357)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1192)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:962)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:898)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:994)
Build Log:
[...truncated 11450 lines...]
[junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/init-core-data-001
[junit4] 2> 1250651 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[E363401D1F6FC7BD]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false)
[junit4] 2> 1250651 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[E363401D1F6FC7BD]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1250656 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1250678 INFO (Thread-3866) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1250678 INFO (Thread-3866) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1250770 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.ZkTestServer start zk server on port:54018
[junit4] 2> 1250770 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1250794 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1250802 INFO (zkCallback-30613-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@228fb82b name:ZooKeeperConnection Watcher:127.0.0.1:54018 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1250802 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1250802 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1250802 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 1250814 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1250826 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1250842 INFO (zkCallback-30614-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6d376033 name:ZooKeeperConnection Watcher:127.0.0.1:54018/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1250842 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1250843 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1250843 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
[junit4] 2> 1250843 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
[junit4] 2> 1250844 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
[junit4] 2> 1250844 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
[junit4] 2> 1250845 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1250845 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
[junit4] 2> 1250845 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 1250846 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
[junit4] 2> 1250846 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1250846 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1250847 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1250847 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
[junit4] 2> 1250847 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1250848 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
[junit4] 2> 1250848 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1250848 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
[junit4] 2> 1250849 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1250849 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
[junit4] 2> 1250849 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1250849 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
[junit4] 2> 1250850 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1250850 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1250850 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1250850 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
[junit4] 2> 1250851 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1250851 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
[junit4] 2> 1250852 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1250866 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1250879 INFO (zkCallback-30615-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@29181fe7 name:ZooKeeperConnection Watcher:127.0.0.1:54018 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1250879 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1250879 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1250879 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 1250880 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1250880 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1250881 INFO (zkCallback-30616-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4db05ded name:ZooKeeperConnection Watcher:127.0.0.1:54018/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1250881 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1250881 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1250881 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
[junit4] 2> 1250881 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.xml
[junit4] 2> 1250882 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
[junit4] 2> 1250883 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/schema.xml
[junit4] 2> 1250883 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf2/enumsConfig.xml
[junit4] 2> 1250883 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/enumsConfig.xml
[junit4] 2> 1250884 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1250884 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1250885 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
[junit4] 2> 1250885 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/stopwords.txt
[junit4] 2> 1250886 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
[junit4] 2> 1250886 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/protwords.txt
[junit4] 2> 1250886 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
[junit4] 2> 1250887 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/currency.xml
[junit4] 2> 1250887 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
[junit4] 2> 1250887 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/open-exchange-rates.json
[junit4] 2> 1250888 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
[junit4] 2> 1250888 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
[junit4] 2> 1250889 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
[junit4] 2> 1250889 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/old_synonyms.txt
[junit4] 2> 1250889 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
[junit4] 2> 1250890 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/synonyms.txt
[junit4] 2> 1250890 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
[junit4] 2> 1250890 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/elevate.xml
[junit4] 2> 1250947 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1
[junit4] 2> 1250948 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.e.j.s.Server jetty-9.3.8.v20160314
[junit4] 2> 1250949 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1fbb6727{/,null,AVAILABLE}
[junit4] 2> 1250974 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.e.j.s.ServerConnector Started ServerConnector@34c8d22{HTTP/1.1,[http/1.1]}{127.0.0.1:33749}
[junit4] 2> 1250974 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.e.j.s.Server Started @1253105ms
[junit4] 2> 1250974 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/tempDir-001/control/data, hostContext=/, hostPort=33749, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores}
[junit4] 2> 1250975 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
[junit4] 2> 1250975 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001'
[junit4] 2> 1250975 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 1250975 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 1250975 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1250980 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1250990 INFO (zkCallback-30617-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@269a429d name:ZooKeeperConnection Watcher:127.0.0.1:54018/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1250990 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1250991 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1250991 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1250991 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/solr.xml
[junit4] 2> 1250995 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores
[junit4] 2> 1250995 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.CoreContainer New CoreContainer 838482464
[junit4] 2> 1250995 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001]
[junit4] 2> 1250995 WARN (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/lib
[junit4] 2> 1250995 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
[junit4] 2> 1250996 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1250996 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
[junit4] 2> 1250996 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
[junit4] 2> 1250996 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54018/solr
[junit4] 2> 1250996 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.ZkController zkHost includes chroot
[junit4] 2> 1250996 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1251006 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1251012 INFO (zkCallback-30620-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2c8961e2 name:ZooKeeperConnection Watcher:127.0.0.1:54018 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1251012 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1251013 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1251014 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$10/1553166551@2eaeef44
[junit4] 2> 1251017 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1251023 INFO (zkCallback-30621-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@661db907 name:ZooKeeperConnection Watcher:127.0.0.1:54018/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1251023 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1251024 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 1251024 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 1251025 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 1251026 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 1251026 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 1251027 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 1251027 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 1251028 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 1251028 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 1251028 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1251029 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1251029 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 1251029 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 1251030 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1251030 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95844247090429958-127.0.0.1:33749_-n_0000000000
[junit4] 2> 1251030 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33749_
[junit4] 2> 1251030 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 1251030 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.Overseer Overseer (id=95844247090429958-127.0.0.1:33749_-n_0000000000) starting
[junit4] 2> 1251031 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 1251033 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1251034 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 1251036 INFO (OverseerCollectionConfigSetProcessor-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 1251039 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33749_
[junit4] 2> 1251039 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:33749_
[junit4] 2> 1251040 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1251040 INFO (zkCallback-30621-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1251040 INFO (zkCallback-30621-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1251047 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
[junit4] 2> 1251047 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.CoreContainer No authentication plugin used.
[junit4] 2> 1251048 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores
[junit4] 2> 1251048 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=control_collection, coreNodeName=, dataDir=data/, shard=}
[junit4] 2> 1251048 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1
[junit4] 2> 1251048 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
[junit4] 2> 1251051 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.ZkController publishing state=down
[junit4] 2> 1251051 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
[junit4] 2> 1251051 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.ZkController Collection control_collection not visible yet, but flagging it so a watch is registered when it becomes visible
[junit4] 2> 1251051 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.c.ZkStateReader addZkWatch [control_collection]
[junit4] 2> 1251052 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.c.ZkStateReader Deleting data for [control_collection]
[junit4] 2> 1251052 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.ZkController look for our core node name
[junit4] 2> 1251053 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:33749",
[junit4] 2> "node_name":"127.0.0.1:33749_",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"down",
[junit4] 2> "shard":null,
[junit4] 2> "collection":"control_collection",
[junit4] 2> "operation":"state"} current state version: 0
[junit4] 2> 1251053 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "core":"collection1",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:33749",
[junit4] 2> "node_name":"127.0.0.1:33749_",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"down",
[junit4] 2> "shard":null,
[junit4] 2> "collection":"control_collection",
[junit4] 2> "operation":"state"}
[junit4] 2> 1251053 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
[junit4] 2> 1251053 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1251054 INFO (zkCallback-30621-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1251054 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1
[junit4] 2> 1251054 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [n:127.0.0.1:33749_ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
[junit4] 2> 1252052 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
[junit4] 2> 1252052 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.ZkController Check for collection zkNode:control_collection
[junit4] 2> 1252052 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.ZkController Collection zkNode exists
[junit4] 2> 1252053 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
[junit4] 2> 1252053 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
[junit4] 2> 1252053 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1'
[junit4] 2> 1252053 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 1252053 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 1252054 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0
[junit4] 2> 1252057 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
[junit4] 2> 1252060 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.1.0
[junit4] 2> 1252086 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
[junit4] 2> 1252090 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1252146 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4] 2> 1252150 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.s.IndexSchema default search field in schema is text
[junit4] 2> 1252151 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.s.IndexSchema unique key field: id
[junit4] 2> 1252155 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
[junit4] 2> 1252156 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
[junit4] 2> 1252157 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1252157 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.s.OpenExchangeRatesOrgProvider Reloading exchange rates from open-exchange-rates.json
[junit4] 2> 1252158 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1
[junit4] 2> 1252158 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
[junit4] 2> 1252158 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
[junit4] 2> 1252158 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1/data/]
[junit4] 2> 1252159 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6815ed1d
[junit4] 2> 1252159 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1/data
[junit4] 2> 1252159 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1/data/index/
[junit4] 2> 1252159 WARN (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
[junit4] 2> 1252159 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1/data/index
[junit4] 2> 1252160 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=23.4150390625, floorSegmentMB=1.994140625, forceMergeDeletesPctAllowed=4.78710904714002, segmentsPerTier=44.0, maxCFSSegmentSizeMB=1.3779296875, noCFSRatio=0.361575544003812
[junit4] 2> 1252183 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
[junit4] 2> commit{dir=MockDirectoryWrapper(SimpleFSDirectory@/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@56497789),segFN=segments_1,generation=1}
[junit4] 2> 1252183 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
[junit4] 2> 1252193 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "nodistrib"
[junit4] 2> 1252193 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
[junit4] 2> 1252194 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4] 2> 1252194 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
[junit4] 2> 1252194 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4] 2> 1252194 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
[junit4] 2> 1252194 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 1252194 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4] 2> 1252194 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
[junit4] 2> 1252194 WARN (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
[junit4] 2> 1252195 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
[junit4] 2> 1252196 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
[junit4] 2> 1252196 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
[junit4] 2> 1252197 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
[junit4] 2> 1252198 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/dump,/update/csv,/sql,/update/json/docs,/admin/luke,/export,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
[junit4] 2> 1252199 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
[junit4] 2> 1252202 INFO (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1/data/
[junit4] 2> 1252202 WARN (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.DirectoryFactory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/control-001/cores/collection1/data/ does not point to a valid data directory; skipping clean-up of old index directories.
[junit4] 2> 1252212 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1252212 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1252213 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1252213 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1252213 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@20f9c9aa
[junit4] 2> 1252214 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
[junit4] 2> commit{dir=MockDirectoryWrapper(SimpleFSDirectory@/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@56497789),segFN=segments_1,generation=1}
[junit4] 2> 1252214 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
[junit4] 2> 1252215 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3d00526b[collection1] main]
[junit4] 2> 1252215 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
[junit4] 2> 1252215 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
[junit4] 2> 1252215 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
[junit4] 2> 1252216 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1252216 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
[junit4] 2> 1252216 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1252216 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
[junit4] 2> 1252216 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1252216 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
[junit4] 2> 1252216 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1252217 INFO (searcherExecutor-7106-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3d00526b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1252217 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
[junit4] 2> 1252217 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
[junit4] 2> 1252217 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
[junit4] 2> 1252217 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1533507955049103360
[junit4] 2> 1252218 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 1.0ms to seed version buckets with highest version 1533507955049103360
[junit4] 2> 1252218 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController watch zkdir /configs/conf1
[junit4] 2> 1252219 INFO (coreLoadExecutor-7105-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.CoreContainer registering core: collection1
[junit4] 2> 1252224 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController Register replica - core:collection1 address:http://127.0.0.1:33749 collection:control_collection shard:shard1
[junit4] 2> 1252224 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/leader_elect/shard1/election
[junit4] 2> 1252235 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.LeaderElector Joined leadership election with path: /collections/control_collection/leader_elect/shard1/election/95844247090429958-core_node1-n_0000000000
[junit4] 2> 1252236 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Running the leader process for shard=shard1 and weAreReplacement=false and leaderVoteWait=180000
[junit4] 2> 1252236 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection"} current state version: 1
[junit4] 2> 1252237 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1252237 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1252237 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:33749/collection1/
[junit4] 2> 1252237 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1252237 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:33749/collection1/ has no replicas
[junit4] 2> 1252239 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/leaders/shard1
[junit4] 2> 1252239 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/95844247090429958-core_node1-n_0000000000
[junit4] 2> 1252240 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:33749/collection1/ shard1
[junit4] 2> 1252240 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "base_url":"http://127.0.0.1:33749",
[junit4] 2> "core":"collection1",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 1252459 INFO (zkCallback-30621-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1252508 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController We are http://127.0.0.1:33749/collection1/ and leader is http://127.0.0.1:33749/collection1/
[junit4] 2> 1252508 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33749
[junit4] 2> 1252508 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1252508 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController publishing state=active
[junit4] 2> 1252508 INFO (coreZkRegister-7099-thread-1-processing-n:127.0.0.1:33749_ x:collection1 s:shard1 c:control_collection r:core_node1) [n:127.0.0.1:33749_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
[junit4] 2> 1252509 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "core":"collection1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:33749",
[junit4] 2> "node_name":"127.0.0.1:33749_",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"active",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "operation":"state"} current state version: 2
[junit4] 2> 1252514 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "core":"collection1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:33749",
[junit4] 2> "node_name":"127.0.0.1:33749_",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"active",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"control_collection",
[junit4] 2> "operation":"state"}
[junit4] 2> 1252555 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.s.i.CloudSolrClient Final constructed zkHost string: 127.0.0.1:54018/solr
[junit4] 2> 1252556 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1252582 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1252609 INFO (zkCallback-30623-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6b532077 name:ZooKeeperConnection Watcher:127.0.0.1:54018/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1252609 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1252609 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1252609 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1252609 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1252610 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1252610 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 1252610 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1252615 INFO (zkCallback-30621-thread-1-processing-n:127.0.0.1:33749_) [n:127.0.0.1:33749_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1252630 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1252630 INFO (zkCallback-30623-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1252643 INFO (zkCallback-30624-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@39379216 name:ZooKeeperConnection Watcher:127.0.0.1:54018/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1252644 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1252644 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1252645 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"create",
[junit4] 2> "name":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "stateFormat":"2"} current state version: 3
[junit4] 2> 1252645 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 1252747 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/shard-1-001/cores/collection1
[junit4] 2> 1252747 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/shard-1-001
[junit4] 2> 1252748 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.e.j.s.Server jetty-9.3.8.v20160314
[junit4] 2> 1252752 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7bc53241{/,null,AVAILABLE}
[junit4] 2> 1252752 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.e.j.s.ServerConnector Started ServerConnector@231ad273{HTTP/1.1,[http/1.1]}{127.0.0.1:44656}
[junit4] 2> 1252753 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.e.j.s.Server Started @1254884ms
[junit4] 2> 1252753 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=44656, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/shard-1-001/cores}
[junit4] 2> 1252753 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
[junit4] 2> 1252753 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/shard-1-001'
[junit4] 2> 1252753 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
[junit4] 2> 1252753 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 1252753 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1252754 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1252754 INFO (zkCallback-30625-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3c8b7886 name:ZooKeeperConnection Watcher:127.0.0.1:54018/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1252754 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1252755 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1252755 INFO (OverseerStateUpdate-95844247090429958-127.0.0.1:33749_-n_0000000000) [n:127.0.0.1:33749_ ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection1/state.json
[junit4] 2> 1252755 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1252755 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/shard-1-001/solr.xml
[junit4] 2> 1252767 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_E363401D1F6FC7BD-001/shard-1-001/cores
[junit4] 2> 1252767 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E363401D1F6FC7BD]) [ ] o.a.s.c.CoreContainer New CoreContainer 1387781136
[junit4] 2> 1252767 INFO (TEST-CollectionsAPIDistributedZkTest.test-seed#[E3634
[...truncated too long message...]
-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/hadoop-common-2.6.0-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/hadoop-hdfs-2.6.0-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/hadoop-minikdc-2.6.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/lucene/build/analysis/icu/lucene-analyzers-icu-6.1.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-antlr.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-regexp.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-resolver.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-commons-net.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jmf.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-oro.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-xalan2.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-commons-logging.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jsch.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-junit4.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-javamail.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jai.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jdepend.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-junit.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-bsf.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-swing.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-netrexx.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-bcel.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-testutil.jar:/home/jenkins/jenkins-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-log4j.jar:/x1/jenkins/jenkins-slave/tools/hudson.model.JDK/latest1.8/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/temp/junit4-J1-20160505_163136_347.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/temp/junit4-J1-20160505_163136_347.suites -stdin
[junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] at java.lang.Thread.run(Thread.java:745)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/build.xml:747: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/build.xml:684: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/common-build.xml:529: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/lucene/common-build.xml:1457: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/lucene/common-build.xml:1014: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
Total time: 245 minutes 24 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No prior successful build to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-NightlyTests-6.x - Build # 58 - Still Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-6.x/58/
7 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test
Error Message:
document count mismatch. control=3941 sum(shards)=3940 cloudClient=3940
Stack Trace:
java.lang.AssertionError: document count mismatch. control=3941 sum(shards)=3940 cloudClient=3940
at __randomizedtesting.SeedInfo.seed([8F44261516A7A083:71019CFB85BCD7B]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1319)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:228)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:992)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:967)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.component.TermVectorComponentDistributedTest
Error Message:
Stack Trace:
java.lang.AssertionError
at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.before(TestRuleTemporaryFilesCleanup.java:112)
at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.before(TestRuleAdapter.java:26)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:35)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.component.TermVectorComponentDistributedTest
Error Message:
71 threads leaked from SUITE scope at org.apache.solr.handler.component.TermVectorComponentDistributedTest: 1) Thread[id=8465, name=coreZkRegister-1049-thread-18-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2) Thread[id=8523, name=Async disk worker #2 for volume /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F44261516A7A083-001/tempDir-001/hdfsBaseDir/data/data3/current, state=TIMED_WAITING, group=FsDatasetAsyncDiskService] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 3) Thread[id=8524, name=Async disk worker #2 for volume /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F44261516A7A083-001/tempDir-001/hdfsBaseDir/data/data1/current, state=TIMED_WAITING, group=FsDatasetAsyncDiskService] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 4) Thread[id=8451, name=coreZkRegister-1049-thread-4-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 5) Thread[id=8468, name=coreZkRegister-1049-thread-21-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 6) Thread[id=8498, name=coreZkRegister-1208-thread-19, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 7) Thread[id=8512, name=updateExecutor-493-thread-6-processing-n:127.0.0.1:50321_ x:awholynewstresscollection_collection4_0_shard1_replica4 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node6, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429) at java.util.concurrent.FutureTask.get(FutureTask.java:191) at org.apache.solr.update.DefaultSolrCoreState$2.run(DefaultSolrCoreState.java:317) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 8) Thread[id=8172, name=java.util.concurrent.ThreadPoolExecutor$Worker@3c2734e6[State = -1, empty queue], state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 9) Thread[id=8475, name=OverseerThreadFactory-1932-thread-4, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 10) Thread[id=8489, name=coreZkRegister-1208-thread-10, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 11) Thread[id=8457, name=coreZkRegister-1049-thread-10, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 12) Thread[id=8494, name=coreZkRegister-1208-thread-15-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1000) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:970) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:926) at org.apache.solr.cloud.ZkController.register(ZkController.java:865) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 13) Thread[id=8438, name=qtp1292668453-8438, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 14) Thread[id=8493, name=coreZkRegister-1208-thread-14-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1000) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:970) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:926) at org.apache.solr.cloud.ZkController.register(ZkController.java:865) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 15) Thread[id=8491, name=coreZkRegister-1208-thread-12-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1000) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:970) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:926) at org.apache.solr.cloud.ZkController.register(ZkController.java:865) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 16) Thread[id=8462, name=coreZkRegister-1049-thread-15-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 17) Thread[id=8418, name=IPC Parameter Sending Thread #38, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 18) Thread[id=8499, name=coreZkRegister-1208-thread-20, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 19) Thread[id=8520, name=zkCallback-460-thread-17, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 20) Thread[id=8455, name=coreZkRegister-1049-thread-8, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 21) Thread[id=8488, name=coreZkRegister-1208-thread-9-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 22) Thread[id=8477, name=OverseerThreadFactory-1932-thread-6, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 23) Thread[id=8444, name=OverseerStateUpdate-95850796072239334-127.0.0.1:50321_-n_0000000006, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:168) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:142) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196) at java.lang.Thread.run(Thread.java:745) 24) Thread[id=8452, name=coreZkRegister-1049-thread-5-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 25) Thread[id=8486, name=coreZkRegister-1208-thread-7-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 26) Thread[id=8443, name=qtp1208954681-8443, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 27) Thread[id=8449, name=zkCallback-465-thread-12, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 28) Thread[id=8476, name=OverseerThreadFactory-1932-thread-5, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 29) Thread[id=8514, name=recoveryExecutor-494-thread-6-processing-n:127.0.0.1:50321_ x:awholynewstresscollection_collection4_0_shard1_replica4 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node6, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:675) at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:663) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:305) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:221) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 30) Thread[id=8484, name=coreZkRegister-1208-thread-5-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1000) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:970) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:926) at org.apache.solr.cloud.ZkController.register(ZkController.java:865) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 31) Thread[id=8474, name=OverseerThreadFactory-1932-thread-3, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 32) Thread[id=8471, name=coreZkRegister-1049-thread-23, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 33) Thread[id=8415, name=IPC Client (2081512138) connection to localhost/127.0.0.1:53213 from jenkins, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:920) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:965) 34) Thread[id=8473, name=OverseerThreadFactory-1932-thread-2, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 35) Thread[id=8464, name=coreZkRegister-1049-thread-17-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 36) Thread[id=8496, name=coreZkRegister-1208-thread-17-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 37) Thread[id=8490, name=coreZkRegister-1208-thread-11, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 38) Thread[id=8417, name=IPC Parameter Sending Thread #37, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 39) Thread[id=8485, name=coreZkRegister-1208-thread-6, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 40) Thread[id=8469, name=coreZkRegister-1049-thread-22-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 41) Thread[id=8441, name=qtp460475988-8441, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 42) Thread[id=8447, name=zkCallback-496-thread-20, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 43) Thread[id=8448, name=coreZkRegister-1049-thread-2-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 44) Thread[id=8459, name=coreZkRegister-1049-thread-12, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 45) Thread[id=8497, name=coreZkRegister-1208-thread-18-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 46) Thread[id=8461, name=coreZkRegister-1049-thread-14-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 47) Thread[id=8467, name=coreZkRegister-1049-thread-20-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 48) Thread[id=8435, name=qtp199004788-8435, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 49) Thread[id=8445, name=OverseerCollectionConfigSetProcessor-95850796072239334-127.0.0.1:50321_-n_0000000006, state=TIMED_WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.getChildren(DistributedQueue.java:342) at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:200) at java.lang.Thread.run(Thread.java:745) 50) Thread[id=8442, name=qtp1208954681-8442, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572) at java.lang.Thread.run(Thread.java:745) 51) Thread[id=8437, name=qtp1292668453-8437, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609) at java.lang.Thread.run(Thread.java:745) 52) Thread[id=8522, name=Async disk worker #2 for volume /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F44261516A7A083-001/tempDir-001/hdfsBaseDir/data/data4/current, state=TIMED_WAITING, group=FsDatasetAsyncDiskService] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 53) Thread[id=8463, name=coreZkRegister-1049-thread-16-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 54) Thread[id=8466, name=coreZkRegister-1049-thread-19, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 55) Thread[id=8521, name=zkCallback-455-thread-15, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 56) Thread[id=8458, name=coreZkRegister-1049-thread-11, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 57) Thread[id=8453, name=coreZkRegister-1049-thread-6-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 58) Thread[id=8454, name=coreZkRegister-1049-thread-7, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 59) Thread[id=8487, name=coreZkRegister-1208-thread-8-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616) at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042) at org.apache.solr.cloud.ZkController.register(ZkController.java:853) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 60) Thread[id=8450, name=coreZkRegister-1049-thread-3, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 61) Thread[id=8495, name=coreZkRegister-1208-thread-16-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1000) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:970) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:926) at org.apache.solr.cloud.ZkController.register(ZkController.java:865) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 62) Thread[id=8492, name=coreZkRegister-1208-thread-13-processing-n:127.0.0.1:50321_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1000) at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:970) at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:926) at org.apache.solr.cloud.ZkController.register(ZkController.java:865) at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 63) Thread[id=8158, name=IPC Client (2081512138) connection to localhost/127.0.0.1:53213 from jenkins, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.io.FilterInputStream.read(FilterInputStream.java:133) at java.io.FilterInputStream.read(FilterInputStream.java:133) at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:513) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) at java.io.DataInputStream.readInt(DataInputStream.java:387) at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1071) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:966) 64) Thread[id=8470, name=zkCallback-465-thread-13, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 65) Thread[id=8483, name=coreZkRegister-1208-thread-4, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 66) Thread[id=8446, name=OverseerHdfsCoreFailoverThread-95850796072239334-127.0.0.1:50321_-n_0000000006, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:138) at java.lang.Thread.run(Thread.java:745) 67) Thread[id=8516, name=updateExecutor-493-thread-7, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 68) Thread[id=8460, name=coreZkRegister-1049-thread-13, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 69) Thread[id=8456, name=coreZkRegister-1049-thread-9, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 70) Thread[id=8518, name=recoveryExecutor-494-thread-7, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 71) Thread[id=8525, name=Async disk worker #2 for volume /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F44261516A7A083-001/tempDir-001/hdfsBaseDir/data/data2/current, state=TIMED_WAITING, group=FsDatasetAsyncDiskService] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 71 threads leaked from SUITE scope at org.apache.solr.handler.component.TermVectorComponentDistributedTest:
1) Thread[id=8465, name=coreZkRegister-1049-thread-18-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616)
at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042)
at org.apache.solr.cloud.ZkController.register(ZkController.java:853)
at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2) Thread[id=8523, name=Async disk worker #2 for volume /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F44261516A7A083-001/tempDir-001/hdfsBaseDir/data/data3/current, state=TIMED_WAITING, group=FsDatasetAsyncDiskService]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
3) Thread[id=8524, name=Async disk worker #2 for volume /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F44261516A7A083-001/tempDir-001/hdfsBaseDir/data/data1/current, state=TIMED_WAITING, group=FsDatasetAsyncDiskService]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
4) Thread[id=8451, name=coreZkRegister-1049-thread-4-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616)
at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042)
at org.apache.solr.cloud.ZkController.register(ZkController.java:853)
at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
5) Thread[id=8468, name=coreZkRegister-1049-thread-21-processing-n:127.0.0.1:37006_, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.ShardLeaderElectionContext.waitForReplicasToComeUp(ElectionContext.java:616)
at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:318)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
at org.apache.solr.cloud.ZkController.joinElection(ZkController.java:1042)
at org.apache.solr.cloud.ZkController.register(ZkController.java:853)
at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:203)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
6) Thread[id=8498, name=coreZkRegister-1208-thread-19, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
7) Thread[id=8512, name=updateExecutor-493-thread-6-processing-n:127.0.0.1:50321_ x:awholynewstresscollection_collection4_0_shard1_replica4 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node6, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429)
at java.util.concurrent.FutureTask.get(FutureTask.java:191)
at org.apache.solr.update.DefaultSolrCoreState$2.run(DefaultSolrCoreState.java:317)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1049045488.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
8) Thread[id=8172, name=java.util.concurrent.ThreadPoolExecutor$Worker@3c2734e6[State = -1, empty queue], state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
9) Threa
[...truncated too long message...]
_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 52) Thread[id=8499, name=coreZkRegister-1208-thread-20, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 53) Thread[id=8458, name=coreZkRegister-1049-thread-11, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 54) Thread[id=8453, name=coreZkRegister-1049-thread-6, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 55) Thread[id=8464, name=coreZkRegister-1049-thread-17, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 56) Thread[id=8518, name=recoveryExecutor-494-thread-7, state=TIMED_WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > 57) Thread[id=8525, name=Async disk worker #2 for volume /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F44261516A7A083-001/tempDir-001/hdfsBaseDir/data/data2/current, state=TIMED_WAITING, group=FsDatasetAsyncDiskService]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:745)
[junit4] > at __randomizedtesting.SeedInfo.seed([8F44261516A7A083]:0)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2833, name=Thread-1066, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap spaceThrowable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2753, name=IPC Server idle connection scanner for port 53213, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
[junit4] > Caused by: java.lang.OutOfMemoryError: Java heap space
[junit4] Completed [519/595 (4!)] on J2 in 8163.44s, 0 tests, 1 failure, 4 errors <<< FAILURES!
[...truncated 229 lines...]
[junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build/solr-core/test/temp/junit4-J2-20160506_203053_666.sysout
[junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
[junit4] java.lang.OutOfMemoryError: Java heap space
[junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/heapdumps/java_pid10577.hprof ...
[junit4] Heap dump file created [682066696 bytes in 5.571 secs]
[junit4] <<< JVM J2: EOF ----
[...truncated 13 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/build.xml:747: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/build.xml:684: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/solr/common-build.xml:529: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/lucene/common-build.xml:1457: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-6.x/lucene/common-build.xml:1014: There were test failures: 595 suites (10 ignored), 2403 tests, 6 suite-level errors, 1 error, 1 failure, 361 ignored (13 assumptions) [seed: 8F44261516A7A083]
Total time: 558 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No prior successful build to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any