You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/05/12 05:00:37 UTC
[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 865 -
Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/865/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
3 tests failed.
FAILED: org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI
Error Message:
Could not find collection : implicitcoll
Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : implicitcoll
at __randomizedtesting.SeedInfo.seed([A321E23AFA287531:C9C06C51C7B2C349]:0)
at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:194)
at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:245)
at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:68)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test
Error Message:
Collection not found: backuprestore
Stack Trace:
org.apache.solr.common.SolrException: Collection not found: backuprestore
at __randomizedtesting.SeedInfo.seed([A321E23AFA287531:2B75DDE054D418C9]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1401)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1094)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1073)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:106)
at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:71)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.indexDocs(AbstractCloudBackupRestoreTestCase.java:203)
at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:113)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.rule.RulesTest.testPortRule
Error Message:
Could not find collection : portRuleColl
Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : portRuleColl
at __randomizedtesting.SeedInfo.seed([A321E23AFA287531:9B81F7A75B788D4F]:0)
at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:194)
at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:245)
at org.apache.solr.cloud.rule.RulesTest.testPortRule(RulesTest.java:106)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11469 lines...]
[junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/init-core-data-001
[junit4] 2> 1052075 INFO (SUITE-CustomCollectionTest-seed#[A321E23AFA287531]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields
[junit4] 2> 1052076 INFO (SUITE-CustomCollectionTest-seed#[A321E23AFA287531]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 1052077 INFO (SUITE-CustomCollectionTest-seed#[A321E23AFA287531]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001
[junit4] 2> 1052077 INFO (SUITE-CustomCollectionTest-seed#[A321E23AFA287531]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1052077 INFO (Thread-1024) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1052077 INFO (Thread-1024) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1052081 ERROR (Thread-1024) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1052181 INFO (SUITE-CustomCollectionTest-seed#[A321E23AFA287531]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:57098
[junit4] 2> 1052216 INFO (jetty-launcher-800-thread-2) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1052216 INFO (jetty-launcher-800-thread-3) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1052216 INFO (jetty-launcher-800-thread-1) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1052216 INFO (jetty-launcher-800-thread-4) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1052237 INFO (jetty-launcher-800-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3560fbc9{/solr,null,AVAILABLE}
[junit4] 2> 1052237 INFO (jetty-launcher-800-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@62834618{/solr,null,AVAILABLE}
[junit4] 2> 1052238 INFO (jetty-launcher-800-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@53bd60ee{HTTP/1.1,[http/1.1]}{127.0.0.1:57101}
[junit4] 2> 1052238 INFO (jetty-launcher-800-thread-1) [ ] o.e.j.s.Server Started @1059746ms
[junit4] 2> 1052238 INFO (jetty-launcher-800-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57101}
[junit4] 2> 1052239 ERROR (jetty-launcher-800-thread-1) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1052239 INFO (jetty-launcher-800-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.7.0
[junit4] 2> 1052239 INFO (jetty-launcher-800-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1052239 INFO (jetty-launcher-800-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1052239 INFO (jetty-launcher-800-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-12T03:47:11.627Z
[junit4] 2> 1052240 INFO (jetty-launcher-800-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4faac321{/solr,null,AVAILABLE}
[junit4] 2> 1052240 INFO (jetty-launcher-800-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2471547{HTTP/1.1,[http/1.1]}{127.0.0.1:57102}
[junit4] 2> 1052241 INFO (jetty-launcher-800-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@cd04f49{/solr,null,AVAILABLE}
[junit4] 2> 1052241 INFO (jetty-launcher-800-thread-2) [ ] o.e.j.s.Server Started @1059748ms
[junit4] 2> 1052241 INFO (jetty-launcher-800-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57102}
[junit4] 2> 1052241 ERROR (jetty-launcher-800-thread-2) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1052241 INFO (jetty-launcher-800-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.7.0
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4679c8a1{HTTP/1.1,[http/1.1]}{127.0.0.1:57103}
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-4) [ ] o.e.j.s.Server Started @1059749ms
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57103}
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-12T03:47:11.630Z
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@42fdf62d{HTTP/1.1,[http/1.1]}{127.0.0.1:57104}
[junit4] 2> 1052242 ERROR (jetty-launcher-800-thread-4) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-3) [ ] o.e.j.s.Server Started @1059750ms
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57104}
[junit4] 2> 1052242 ERROR (jetty-launcher-800-thread-3) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1052242 INFO (jetty-launcher-800-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.7.0
[junit4] 2> 1052243 INFO (jetty-launcher-800-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1052243 INFO (jetty-launcher-800-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1052243 INFO (jetty-launcher-800-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-12T03:47:11.631Z
[junit4] 2> 1052243 INFO (jetty-launcher-800-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 6.7.0
[junit4] 2> 1052243 INFO (jetty-launcher-800-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1052243 INFO (jetty-launcher-800-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1052243 INFO (jetty-launcher-800-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-12T03:47:11.631Z
[junit4] 2> 1052247 INFO (jetty-launcher-800-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1052248 INFO (jetty-launcher-800-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1052249 INFO (jetty-launcher-800-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1052249 INFO (jetty-launcher-800-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1052261 INFO (jetty-launcher-800-thread-3) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1052261 INFO (jetty-launcher-800-thread-2) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1052262 INFO (jetty-launcher-800-thread-1) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1052262 INFO (jetty-launcher-800-thread-4) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
[junit4] 2> 1052263 INFO (jetty-launcher-800-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57098/solr
[junit4] 2> 1052264 INFO (jetty-launcher-800-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57098/solr
[junit4] 2> 1052265 INFO (jetty-launcher-800-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57098/solr
[junit4] 2> 1052271 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15bfac477ec0006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1052272 INFO (jetty-launcher-800-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57098/solr
[junit4] 2> 1052329 INFO (jetty-launcher-800-thread-3) [n:127.0.0.1:57104_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1052330 INFO (jetty-launcher-800-thread-4) [n:127.0.0.1:57103_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1052331 INFO (jetty-launcher-800-thread-3) [n:127.0.0.1:57104_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57104_solr
[junit4] 2> 1052332 INFO (jetty-launcher-800-thread-2) [n:127.0.0.1:57102_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1052332 INFO (jetty-launcher-800-thread-4) [n:127.0.0.1:57103_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57103_solr
[junit4] 2> 1052333 INFO (jetty-launcher-800-thread-3) [n:127.0.0.1:57104_solr ] o.a.s.c.Overseer Overseer (id=97947538651086857-127.0.0.1:57104_solr-n_0000000000) starting
[junit4] 2> 1052336 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1052338 INFO (zkCallback-821-thread-1-processing-n:127.0.0.1:57103_solr) [n:127.0.0.1:57103_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1052338 INFO (zkCallback-817-thread-1-processing-n:127.0.0.1:57102_solr) [n:127.0.0.1:57102_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1052341 INFO (jetty-launcher-800-thread-2) [n:127.0.0.1:57102_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57102_solr
[junit4] 2> 1052345 INFO (zkCallback-817-thread-1-processing-n:127.0.0.1:57102_solr) [n:127.0.0.1:57102_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1052345 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1052345 INFO (zkCallback-821-thread-1-processing-n:127.0.0.1:57103_solr) [n:127.0.0.1:57103_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1052354 INFO (jetty-launcher-800-thread-3) [n:127.0.0.1:57104_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57104_solr
[junit4] 2> 1052357 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1052357 INFO (zkCallback-821-thread-1-processing-n:127.0.0.1:57103_solr) [n:127.0.0.1:57103_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1052357 INFO (zkCallback-817-thread-1-processing-n:127.0.0.1:57102_solr) [n:127.0.0.1:57102_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1052383 INFO (jetty-launcher-800-thread-1) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1052399 INFO (jetty-launcher-800-thread-1) [n:127.0.0.1:57101_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1052402 INFO (jetty-launcher-800-thread-1) [n:127.0.0.1:57101_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57101_solr
[junit4] 2> 1052407 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1052410 INFO (zkCallback-821-thread-1-processing-n:127.0.0.1:57103_solr) [n:127.0.0.1:57103_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1052410 INFO (zkCallback-817-thread-1-processing-n:127.0.0.1:57102_solr) [n:127.0.0.1:57102_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1052410 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1052522 INFO (jetty-launcher-800-thread-2) [n:127.0.0.1:57102_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node2/.
[junit4] 2> 1052540 INFO (jetty-launcher-800-thread-4) [n:127.0.0.1:57103_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/.
[junit4] 2> 1052614 INFO (jetty-launcher-800-thread-1) [n:127.0.0.1:57101_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/.
[junit4] 2> 1052615 INFO (jetty-launcher-800-thread-3) [n:127.0.0.1:57104_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/.
[junit4] 2> 1052667 INFO (SUITE-CustomCollectionTest-seed#[A321E23AFA287531]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1052669 INFO (SUITE-CustomCollectionTest-seed#[A321E23AFA287531]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57098/solr ready
[junit4] 2> 1052741 INFO (TEST-CustomCollectionTest.testCreateShardRepFactor-seed#[A321E23AFA287531]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCreateShardRepFactor
[junit4] 2> 1052744 INFO (TEST-CustomCollectionTest.testCreateShardRepFactor-seed#[A321E23AFA287531]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1052753 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params shards=a,b&replicationFactor=1&collection.configName=conf&name=testCreateShardRepFactor&router.name=implicit&action=CREATE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1052756 INFO (OverseerThreadFactory-2254-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.CreateCollectionCmd Create collection testCreateShardRepFactor
[junit4] 2> 1052881 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=testCreateShardRepFactor_b_replica1&action=CREATE&numShards=2&collection=testCreateShardRepFactor&shard=b&wt=javabin&version=2
[junit4] 2> 1052881 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=testCreateShardRepFactor_a_replica1&action=CREATE&numShards=2&collection=testCreateShardRepFactor&shard=a&wt=javabin&version=2
[junit4] 2> 1052882 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1052882 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1053002 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1053002 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1053914 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1053915 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1053936 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_a_replica1] Schema name=cloud-dynamic
[junit4] 2> 1053937 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_b_replica1] Schema name=cloud-dynamic
[junit4] 2> 1053979 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1053979 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.CoreContainer Creating SolrCore 'testCreateShardRepFactor_a_replica1' using configuration from collection testCreateShardRepFactor, trusted=true
[junit4] 2> 1053979 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1053979 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.SolrCore [[testCreateShardRepFactor_a_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/testCreateShardRepFactor_a_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/./testCreateShardRepFactor_a_replica1/data/]
[junit4] 2> 1053984 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1053984 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.CoreContainer Creating SolrCore 'testCreateShardRepFactor_b_replica1' using configuration from collection testCreateShardRepFactor, trusted=true
[junit4] 2> 1053985 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1053985 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.SolrCore [[testCreateShardRepFactor_b_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/testCreateShardRepFactor_b_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/./testCreateShardRepFactor_b_replica1/data/]
[junit4] 2> 1054079 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1054079 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1054080 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1054080 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1054091 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1054091 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1054092 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1054092 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1054092 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4e7914a2[testCreateShardRepFactor_b_replica1] main]
[junit4] 2> 1054094 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@bbf135[testCreateShardRepFactor_a_replica1] main]
[junit4] 2> 1054094 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1054095 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1054095 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1054095 INFO (searcherExecutor-2264-thread-1-processing-n:127.0.0.1:57104_solr x:testCreateShardRepFactor_b_replica1 s:b c:testCreateShardRepFactor) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.SolrCore [testCreateShardRepFactor_b_replica1] Registered new searcher Searcher@4e7914a2[testCreateShardRepFactor_b_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1054095 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1054095 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567160620530270208
[junit4] 2> 1054096 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1054096 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1054096 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567160620531318784
[junit4] 2> 1054096 INFO (searcherExecutor-2263-thread-1-processing-n:127.0.0.1:57101_solr x:testCreateShardRepFactor_a_replica1 s:a c:testCreateShardRepFactor) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.SolrCore [testCreateShardRepFactor_a_replica1] Registered new searcher Searcher@bbf135[testCreateShardRepFactor_a_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1054106 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1054106 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1054106 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57101/solr/testCreateShardRepFactor_a_replica1/
[junit4] 2> 1054106 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1054106 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:57101/solr/testCreateShardRepFactor_a_replica1/ has no replicas
[junit4] 2> 1054106 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1054107 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1054107 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1054107 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57104/solr/testCreateShardRepFactor_b_replica1/
[junit4] 2> 1054107 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1054108 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:57104/solr/testCreateShardRepFactor_b_replica1/ has no replicas
[junit4] 2> 1054108 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1054115 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57101/solr/testCreateShardRepFactor_a_replica1/ a
[junit4] 2> 1054117 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57104/solr/testCreateShardRepFactor_b_replica1/ b
[junit4] 2> 1054229 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1054229 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1054270 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1054277 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1054285 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:testCreateShardRepFactor s:b x:testCreateShardRepFactor_b_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=testCreateShardRepFactor_b_replica1&action=CREATE&numShards=2&collection=testCreateShardRepFactor&shard=b&wt=javabin&version=2} status=0 QTime=1403
[junit4] 2> 1054285 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:testCreateShardRepFactor s:a x:testCreateShardRepFactor_a_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=testCreateShardRepFactor_a_replica1&action=CREATE&numShards=2&collection=testCreateShardRepFactor&shard=a&wt=javabin&version=2} status=0 QTime=1403
[junit4] 2> 1054291 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1054393 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1054393 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1054770 INFO (OverseerCollectionConfigSetProcessor-97947538651086857-127.0.0.1:57104_solr-n_0000000000) [n:127.0.0.1:57104_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1055294 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={shards=a,b&replicationFactor=1&collection.configName=conf&name=testCreateShardRepFactor&router.name=implicit&action=CREATE&wt=javabin&version=2} status=0 QTime=2542
[junit4] 2> 1055298 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :createshard with params action=CREATESHARD&collection=testCreateShardRepFactor&shard=x&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1055304 INFO (OverseerThreadFactory-2254-thread-2-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.CreateShardCmd Create shard invoked: {
[junit4] 2> "collection":"testCreateShardRepFactor",
[junit4] 2> "shard":"x",
[junit4] 2> "operation":"createshard"}
[junit4] 2> 1055417 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1055417 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1055517 INFO (OverseerThreadFactory-2254-thread-2-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.CreateShardCmd Creating shard testCreateShardRepFactor_x_replica1 as part of slice x of collection testCreateShardRepFactor on 127.0.0.1:57103_solr
[junit4] 2> 1055520 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&name=testCreateShardRepFactor_x_replica1&action=CREATE&numShards=1&collection=testCreateShardRepFactor&shard=x&wt=javabin&version=2
[junit4] 2> 1055521 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1055636 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1055636 INFO (zkCallback-821-thread-1-processing-n:127.0.0.1:57103_solr) [n:127.0.0.1:57103_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1055636 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1056566 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1056581 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.s.IndexSchema [testCreateShardRepFactor_x_replica1] Schema name=cloud-dynamic
[junit4] 2> 1056668 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1056668 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.CoreContainer Creating SolrCore 'testCreateShardRepFactor_x_replica1' using configuration from collection testCreateShardRepFactor, trusted=true
[junit4] 2> 1056668 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1056669 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.SolrCore [[testCreateShardRepFactor_x_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/testCreateShardRepFactor_x_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/./testCreateShardRepFactor_x_replica1/data/]
[junit4] 2> 1056828 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1056828 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1056832 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1056832 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1056835 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1a367eaf[testCreateShardRepFactor_x_replica1] main]
[junit4] 2> 1056838 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 1056839 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 1056839 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1056839 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1567160623407562752
[junit4] 2> 1056839 INFO (searcherExecutor-2273-thread-1-processing-n:127.0.0.1:57103_solr x:testCreateShardRepFactor_x_replica1 s:x c:testCreateShardRepFactor) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.SolrCore [testCreateShardRepFactor_x_replica1] Registered new searcher Searcher@1a367eaf[testCreateShardRepFactor_x_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1056850 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1056850 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1056850 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57103/solr/testCreateShardRepFactor_x_replica1/
[junit4] 2> 1056850 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1056850 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.SyncStrategy http://127.0.0.1:57103/solr/testCreateShardRepFactor_x_replica1/ has no replicas
[junit4] 2> 1056850 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1056860 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57103/solr/testCreateShardRepFactor_x_replica1/ x
[junit4] 2> 1056971 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1056972 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1056972 INFO (zkCallback-821-thread-1-processing-n:127.0.0.1:57103_solr) [n:127.0.0.1:57103_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1057022 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1057028 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:testCreateShardRepFactor s:x x:testCreateShardRepFactor_x_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&name=testCreateShardRepFactor_x_replica1&action=CREATE&numShards=1&collection=testCreateShardRepFactor&shard=x&wt=javabin&version=2} status=0 QTime=1507
[junit4] 2> 1057029 INFO (OverseerThreadFactory-2254-thread-2-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.CreateShardCmd Finished create command on all shards for collection: testCreateShardRepFactor
[junit4] 2> 1057038 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=CREATESHARD&collection=testCreateShardRepFactor&shard=x&wt=javabin&version=2} status=0 QTime=1739
[junit4] 2> 1057040 INFO (TEST-CustomCollectionTest.testCreateShardRepFactor-seed#[A321E23AFA287531]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testCreateShardRepFactor
[junit4] 2> 1057100 INFO (TEST-CustomCollectionTest.testRouteFieldForImplicitRouter-seed#[A321E23AFA287531]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRouteFieldForImplicitRouter
[junit4] 2> 1057104 INFO (TEST-CustomCollectionTest.testRouteFieldForImplicitRouter-seed#[A321E23AFA287531]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1057109 INFO (qtp1566842664-4990) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=testCreateShardRepFactor&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1057116 INFO (OverseerThreadFactory-2254-thread-3-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 1057116 INFO (OverseerCollectionConfigSetProcessor-97947538651086857-127.0.0.1:57104_solr-n_0000000000) [n:127.0.0.1:57104_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1057120 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.b.replica1
[junit4] 2> 1057120 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.a.replica1
[junit4] 2> 1057121 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr ] o.a.s.c.SolrCore [testCreateShardRepFactor_b_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@4a99011f
[junit4] 2> 1057121 INFO (qtp1566842664-4991) [n:127.0.0.1:57103_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.x.replica1
[junit4] 2> 1057122 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr ] o.a.s.c.SolrCore [testCreateShardRepFactor_a_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@498b9324
[junit4] 2> 1057122 INFO (qtp1566842664-4991) [n:127.0.0.1:57103_solr ] o.a.s.c.SolrCore [testCreateShardRepFactor_x_replica1] CLOSING SolrCore org.apache.solr.core.SolrCore@30cabefa
[junit4] 2> 1057133 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.a.replica1
[junit4] 2> 1057134 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.b.replica1
[junit4] 2> 1057135 INFO (qtp1566842664-4991) [n:127.0.0.1:57103_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.testCreateShardRepFactor.x.replica1
[junit4] 2> 1057136 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1057136 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1057136 INFO (zkCallback-821-thread-1-processing-n:127.0.0.1:57103_solr) [n:127.0.0.1:57103_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCreateShardRepFactor/state.json] for collection [testCreateShardRepFactor] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1057154 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=testCreateShardRepFactor_a_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=34
[junit4] 2> 1057157 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=testCreateShardRepFactor_b_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=37
[junit4] 2> 1057157 INFO (qtp1566842664-4991) [n:127.0.0.1:57103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=testCreateShardRepFactor_x_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=36
[junit4] 2> 1057886 INFO (qtp1566842664-4990) [n:127.0.0.1:57103_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testCreateShardRepFactor&action=DELETE&wt=javabin&version=2} status=0 QTime=777
[junit4] 2> 1057918 INFO (qtp1566842664-4992) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params shards=a,b,c,d&replicationFactor=3&collection.configName=conf&maxShardsPerNode=4&router.field=shard_s&name=withShardField&router.name=implicit&action=CREATE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1057923 INFO (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.CreateCollectionCmd Create collection withShardField
[junit4] 2> 1057925 INFO (OverseerCollectionConfigSetProcessor-97947538651086857-127.0.0.1:57104_solr-n_0000000000) [n:127.0.0.1:57104_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1058182 INFO (qtp369299731-4977) [n:127.0.0.1:57101_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_c_replica1&action=CREATE&numShards=4&collection=withShardField&shard=c&wt=javabin&version=2
[junit4] 2> 1058183 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_d_replica2&action=CREATE&numShards=4&collection=withShardField&shard=d&wt=javabin&version=2
[junit4] 2> 1058185 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_d_replica3&action=CREATE&numShards=4&collection=withShardField&shard=d&wt=javabin&version=2
[junit4] 2> 1058185 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_b_replica1&action=CREATE&numShards=4&collection=withShardField&shard=b&wt=javabin&version=2
[junit4] 2> 1058186 INFO (qtp665309728-4965) [n:127.0.0.1:57104_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_b_replica3&action=CREATE&numShards=4&collection=withShardField&shard=b&wt=javabin&version=2
[junit4] 2> 1058192 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_c_replica2&action=CREATE&numShards=4&collection=withShardField&shard=c&wt=javabin&version=2
[junit4] 2> 1058197 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_a_replica2&action=CREATE&numShards=4&collection=withShardField&shard=a&wt=javabin&version=2
[junit4] 2> 1058203 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_a_replica3&action=CREATE&numShards=4&collection=withShardField&shard=a&wt=javabin&version=2
[junit4] 2> 1058206 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_d_replica1&action=CREATE&numShards=4&collection=withShardField&shard=d&wt=javabin&version=2
[junit4] 2> 1058211 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_a_replica1&action=CREATE&numShards=4&collection=withShardField&shard=a&wt=javabin&version=2
[junit4] 2> 1058211 INFO (qtp959166480-4970) [n:127.0.0.1:57102_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_b_replica2&action=CREATE&numShards=4&collection=withShardField&shard=b&wt=javabin&version=2
[junit4] 2> 1058212 INFO (qtp959166480-4970) [n:127.0.0.1:57102_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1058213 INFO (qtp959166480-4971) [n:127.0.0.1:57102_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=withShardField_c_replica3&action=CREATE&numShards=4&collection=withShardField&shard=c&wt=javabin&version=2
[junit4] 2> 1058375 INFO (zkCallback-818-thread-1-processing-n:127.0.0.1:57104_solr) [n:127.0.0.1:57104_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1058376 INFO (zkCallback-822-thread-1-processing-n:127.0.0.1:57101_solr) [n:127.0.0.1:57101_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1058376 INFO (zkCallback-821-thread-1-processing-n:127.0.0.1:57103_solr) [n:127.0.0.1:57103_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1058376 INFO (zkCallback-817-thread-1-processing-n:127.0.0.1:57102_solr) [n:127.0.0.1:57102_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/withShardField/state.json] for collection [withShardField] has occurred - updating... (live nodes size: [4])
[junit4] 2> 1059316 INFO (qtp665309728-4965) [n:127.0.0.1:57104_solr c:withShardField s:b x:withShardField_b_replica3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059316 INFO (qtp369299731-4977) [n:127.0.0.1:57101_solr c:withShardField s:c x:withShardField_c_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059317 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr c:withShardField s:a x:withShardField_a_replica3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059363 INFO (qtp959166480-4970) [n:127.0.0.1:57102_solr c:withShardField s:b x:withShardField_b_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059365 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr c:withShardField s:d x:withShardField_d_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059367 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:withShardField s:b x:withShardField_b_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059373 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr c:withShardField s:d x:withShardField_d_replica3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059374 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059375 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr c:withShardField s:c x:withShardField_c_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059397 INFO (qtp959166480-4971) [n:127.0.0.1:57102_solr c:withShardField s:c x:withShardField_c_replica3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059404 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:withShardField s:b x:withShardField_b_replica1] o.a.s.s.IndexSchema [withShardField_b_replica1] Schema name=cloud-dynamic
[junit4] 2> 1059405 INFO (qtp369299731-4977) [n:127.0.0.1:57101_solr c:withShardField s:c x:withShardField_c_replica1] o.a.s.s.IndexSchema [withShardField_c_replica1] Schema name=cloud-dynamic
[junit4] 2> 1059406 INFO (qtp665309728-4965) [n:127.0.0.1:57104_solr c:withShardField s:b x:withShardField_b_replica3] o.a.s.s.IndexSchema [withShardField_b_replica3] Schema name=cloud-dynamic
[junit4] 2> 1059412 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr c:withShardField s:d x:withShardField_d_replica1] o.a.s.s.IndexSchema [withShardField_d_replica1] Schema name=cloud-dynamic
[junit4] 2> 1059419 INFO (qtp959166480-4970) [n:127.0.0.1:57102_solr c:withShardField s:b x:withShardField_b_replica2] o.a.s.s.IndexSchema [withShardField_b_replica2] Schema name=cloud-dynamic
[junit4] 2> 1059437 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr c:withShardField s:c x:withShardField_c_replica2] o.a.s.s.IndexSchema [withShardField_c_replica2] Schema name=cloud-dynamic
[junit4] 2> 1059459 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:withShardField s:a x:withShardField_a_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059461 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:withShardField s:d x:withShardField_d_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.7.0
[junit4] 2> 1059468 INFO (qtp959166480-4971) [n:127.0.0.1:57102_solr c:withShardField s:c x:withShardField_c_replica3] o.a.s.s.IndexSchema [withShardField_c_replica3] Schema name=cloud-dynamic
[junit4] 2> 1059470 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr c:withShardField s:a x:withShardField_a_replica3] o.a.s.s.IndexSchema [withShardField_a_replica3] Schema name=cloud-dynamic
[junit4] 2> 1059535 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:withShardField s:d x:withShardField_d_replica2] o.a.s.s.IndexSchema [withShardField_d_replica2] Schema name=cloud-dynamic
[junit4] 2> 1059538 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.s.IndexSchema [withShardField_a_replica1] Schema name=cloud-dynamic
[junit4] 2> 1059541 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:withShardField s:a x:withShardField_a_replica2] o.a.s.s.IndexSchema [withShardField_a_replica2] Schema name=cloud-dynamic
[junit4] 2> 1059551 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr c:withShardField s:d x:withShardField_d_replica3] o.a.s.s.IndexSchema [withShardField_d_replica3] Schema name=cloud-dynamic
[junit4] 2> 1059633 INFO (qtp959166480-4970) [n:127.0.0.1:57102_solr c:withShardField s:b x:withShardField_b_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059633 INFO (qtp959166480-4970) [n:127.0.0.1:57102_solr c:withShardField s:b x:withShardField_b_replica2] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_b_replica2' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059634 INFO (qtp959166480-4970) [n:127.0.0.1:57102_solr c:withShardField s:b x:withShardField_b_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059634 INFO (qtp959166480-4970) [n:127.0.0.1:57102_solr c:withShardField s:b x:withShardField_b_replica2] o.a.s.c.SolrCore [[withShardField_b_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node2/withShardField_b_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node2/./withShardField_b_replica2/data/]
[junit4] 2> 1059656 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr c:withShardField s:a x:withShardField_a_replica3] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059656 INFO (qtp959166480-4971) [n:127.0.0.1:57102_solr c:withShardField s:c x:withShardField_c_replica3] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059656 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr c:withShardField s:a x:withShardField_a_replica3] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_a_replica3' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059716 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr c:withShardField s:a x:withShardField_a_replica3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059716 INFO (qtp369299731-4976) [n:127.0.0.1:57101_solr c:withShardField s:a x:withShardField_a_replica3] o.a.s.c.SolrCore [[withShardField_a_replica3] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/withShardField_a_replica3], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/./withShardField_a_replica3/data/]
[junit4] 2> 1059712 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:withShardField s:b x:withShardField_b_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059718 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:withShardField s:b x:withShardField_b_replica1] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_b_replica1' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059719 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:withShardField s:b x:withShardField_b_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059719 INFO (qtp1566842664-4989) [n:127.0.0.1:57103_solr c:withShardField s:b x:withShardField_b_replica1] o.a.s.c.SolrCore [[withShardField_b_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/withShardField_b_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/./withShardField_b_replica1/data/]
[junit4] 2> 1059711 INFO (qtp369299731-4977) [n:127.0.0.1:57101_solr c:withShardField s:c x:withShardField_c_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059722 INFO (qtp369299731-4977) [n:127.0.0.1:57101_solr c:withShardField s:c x:withShardField_c_replica1] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_c_replica1' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059710 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059722 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_a_replica1' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059705 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr c:withShardField s:d x:withShardField_d_replica1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059722 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr c:withShardField s:d x:withShardField_d_replica1] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_d_replica1' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059703 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:withShardField s:a x:withShardField_a_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059722 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:withShardField s:a x:withShardField_a_replica2] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_a_replica2' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059722 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059723 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.c.SolrCore [[withShardField_a_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node2/withShardField_a_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node2/./withShardField_a_replica1/data/]
[junit4] 2> 1059760 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:withShardField s:d x:withShardField_d_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059760 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:withShardField s:d x:withShardField_d_replica2] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_d_replica2' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059761 INFO (qtp369299731-4977) [n:127.0.0.1:57101_solr c:withShardField s:c x:withShardField_c_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059761 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:withShardField s:d x:withShardField_d_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059761 INFO (qtp369299731-4977) [n:127.0.0.1:57101_solr c:withShardField s:c x:withShardField_c_replica1] o.a.s.c.SolrCore [[withShardField_c_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/withShardField_c_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/./withShardField_c_replica1/data/]
[junit4] 2> 1059761 INFO (qtp369299731-4975) [n:127.0.0.1:57101_solr c:withShardField s:d x:withShardField_d_replica2] o.a.s.c.SolrCore [[withShardField_d_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/withShardField_d_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node1/./withShardField_d_replica2/data/]
[junit4] 2> 1059691 INFO (qtp665309728-4965) [n:127.0.0.1:57104_solr c:withShardField s:b x:withShardField_b_replica3] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059761 INFO (qtp665309728-4965) [n:127.0.0.1:57104_solr c:withShardField s:b x:withShardField_b_replica3] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_b_replica3' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059762 INFO (qtp665309728-4965) [n:127.0.0.1:57104_solr c:withShardField s:b x:withShardField_b_replica3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059762 INFO (qtp665309728-4965) [n:127.0.0.1:57104_solr c:withShardField s:b x:withShardField_b_replica3] o.a.s.c.SolrCore [[withShardField_b_replica3] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/withShardField_b_replica3], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/./withShardField_b_replica3/data/]
[junit4] 2> 1059656 INFO (qtp959166480-4971) [n:127.0.0.1:57102_solr c:withShardField s:c x:withShardField_c_replica3] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_c_replica3' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059764 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr c:withShardField s:c x:withShardField_c_replica2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059764 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr c:withShardField s:c x:withShardField_c_replica2] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_c_replica2' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059764 INFO (qtp959166480-4971) [n:127.0.0.1:57102_solr c:withShardField s:c x:withShardField_c_replica3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059764 INFO (qtp959166480-4971) [n:127.0.0.1:57102_solr c:withShardField s:c x:withShardField_c_replica3] o.a.s.c.SolrCore [[withShardField_c_replica3] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node2/withShardField_c_replica3], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node2/./withShardField_c_replica3/data/]
[junit4] 2> 1059765 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr c:withShardField s:c x:withShardField_c_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059765 INFO (qtp1566842664-4988) [n:127.0.0.1:57103_solr c:withShardField s:c x:withShardField_c_replica2] o.a.s.c.SolrCore [[withShardField_c_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/withShardField_c_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/./withShardField_c_replica2/data/]
[junit4] 2> 1059766 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:withShardField s:a x:withShardField_a_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059766 INFO (qtp665309728-4962) [n:127.0.0.1:57104_solr c:withShardField s:a x:withShardField_a_replica2] o.a.s.c.SolrCore [[withShardField_a_replica2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/withShardField_a_replica2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/./withShardField_a_replica2/data/]
[junit4] 2> 1059769 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr c:withShardField s:d x:withShardField_d_replica3] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
[junit4] 2> 1059769 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr c:withShardField s:d x:withShardField_d_replica3] o.a.s.c.CoreContainer Creating SolrCore 'withShardField_d_replica3' using configuration from collection withShardField, trusted=true
[junit4] 2> 1059769 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr c:withShardField s:d x:withShardField_d_replica3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059770 INFO (qtp1566842664-4987) [n:127.0.0.1:57103_solr c:withShardField s:d x:withShardField_d_replica3] o.a.s.c.SolrCore [[withShardField_d_replica3] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/withShardField_d_replica3], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node4/./withShardField_d_replica3/data/]
[junit4] 2> 1059723 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr c:withShardField s:d x:withShardField_d_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1059770 INFO (qtp665309728-4963) [n:127.0.0.1:57104_solr c:withShardField s:d x:withShardField_d_replica1] o.a.s.c.SolrCore [[withShardField_d_replica1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/withShardField_d_replica1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_A321E23AFA287531-001/tempDir-001/node3/./withShardField_d_replica1/data/]
[junit4] 2> 1060109 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1060109 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1060153 INFO (qtp959166480-4960) [n:127.0.0.1:57102_solr c:withShardField s:a x:withShardField_a_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1060154 INFO (qtp95916648
[...truncated too long message...]
rker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64167 64167
[junit4] 2> 3739087 INFO (Thread-4381) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64167 64167
[junit4] 2> 3749815 WARN (Thread-4381) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 12 /solr/aliases.json
[junit4] 2> 7 /solr/configs/conf
[junit4] 2> 5 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 12 /solr/clusterprops.json
[junit4] 2> 12 /solr/clusterstate.json
[junit4] 2> 5 /solr/collections/rulesColl/state.json
[junit4] 2> 2 /solr/collections/hostFragment/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 26 /solr/collections
[junit4] 2> 12 /solr/live_nodes
[junit4] 2>
[junit4] 2> 3749815 INFO (SUITE-RulesTest-seed#[A321E23AFA287531]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.rule.RulesTest_A321E23AFA287531-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=es-US, timezone=Europe/Kaliningrad
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_131 (64-bit)/cpus=3,threads=1,free=48585560,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [SyncSliceTest, TestApiFramework, BlockJoinFacetDistribTest, SolrCoreTest, BlockJoinFacetSimpleTest, TestFieldTypeCollectionResource, TestLRUCache, TestMaxScoreQueryParser, TestReplicaProperties, HighlighterTest, CollectionsAPIAsyncDistributedZkTest, BasicDistributedZkTest, TestScoreJoinQPNoScore, TestLeaderElectionWithEmptyReplica, UnloadDistributedZkTest, SuggesterFSTTest, SuggestComponentContextFilterQueryTest, TestRandomRequestDistribution, AssignTest, TestStressVersions, BadComponentTest, TestBM25SimilarityFactory, SimpleFacetsTest, QueryEqualityTest, SaslZkACLProviderTest, SynonymTokenizerTest, ClassificationUpdateProcessorFactoryTest, LeaderFailoverAfterPartitionTest, PolyFieldTest, SolrMetricReporterTest, TestConfigSetsAPIExclusivity, TestSolrDynamicMBean, TestFastOutputStream, HdfsBasicDistributedZk2Test, TestFilteredDocIdSet, TestSolrDeletionPolicy1, TestStressCloudBlindAtomicUpdates, HdfsThreadLeakTest, UUIDUpdateProcessorFallbackTest, SolrGangliaReporterTest, TestDistributedSearch, DocumentAnalysisRequestHandlerTest, SegmentsInfoRequestHandlerTest, DirectUpdateHandlerTest, TestMacroExpander, TestStressLucene, TestJsonFacetRefinement, DisMaxRequestHandlerTest, TestCollectionAPI, TestSegmentSorting, TestJettySolrRunner, HdfsUnloadDistributedZkTest, TestFileDictionaryLookup, RollingRestartTest, ConjunctionSolrSpellCheckerTest, TestFieldCacheVsDocValues, TestConfigSetImmutable, TestRestManager, TestSurroundQueryParser, TestExclusionRuleCollectionAccess, TestSolrCloudWithKerberosAlt, InfixSuggestersTest, SolrCoreMetricManagerTest, ZkControllerTest, JSONWriterTest, CollectionStateFormat2Test, ClassificationUpdateProcessorTest, TestSolrFieldCacheMBean, TestSolrJ, CustomCollectionTest, TestXIncludeConfig, TestFieldCacheSort, TestConfigsApi, CloudExitableDirectoryReaderTest, TriLevelCompositeIdRoutingTest, TestGraphMLResponseWriter, TestExportWriter, ShardRoutingTest, HttpSolrCallGetCoreTest, CdcrUpdateLogTest, RecoveryZkTest, InfoHandlerTest, ManagedSchemaRoundRobinCloudTest, DirectSolrConnectionTest, OverriddenZkACLAndCredentialsProvidersTest, SoftAutoCommitTest, ConfigureRecoveryStrategyTest, HdfsChaosMonkeyNothingIsSafeTest, PeerSyncReplicationTest, TestCloudNestedDocsSort, TestRTGBase, XsltUpdateRequestHandlerTest, DistributedFacetExistsSmallTest, TestNumericTerms64, SpatialRPTFieldTypeTest, TestPointFields, HLLUtilTest, TestFaceting, SharedFSAutoReplicaFailoverTest, StandardRequestHandlerTest, SimpleMLTQParserTest, SimplePostToolTest, BigEndianAscendingWordDeserializerTest, TestDistribDocBasedVersion, RemoteQueryErrorTest, TestCollationField, VersionInfoTest, ConnectionManagerTest, DOMUtilTest, XmlUpdateRequestHandlerTest, TestSha256AuthenticationProvider, ChangedSchemaMergeTest, TestAnalyzedSuggestions, SolrJmxReporterTest, TestHdfsUpdateLog, DistributedSuggestComponentTest, DocValuesMissingTest, MultiTermTest, PrimitiveFieldTypeTest, OverseerModifyCollectionTest, TestSimpleQParserPlugin, TestUseDocValuesAsStored2, TestCSVLoader, WrapperMergePolicyFactoryTest, TestGroupingSearch, AnalyticsQueryTest, TestSerializedLuceneMatchVersion, HttpPartitionTest, DistributedFacetPivotSmallTest, SimpleCollectionCreateDeleteTest, TestSchemaNameResource, TestRemoteStreaming, JavabinLoaderTest, TestRandomCollapseQParserPlugin, TestManagedSchemaAPI, TestPerFieldSimilarityClassic, StressHdfsTest, TestInitParams, BadCopyFieldTest, TestExtendedDismaxParser, FullHLLTest, TestCSVResponseWriter, TestRTimerTree, PropertiesRequestHandlerTest, TestBinaryField, HdfsCollectionsAPIDistributedZkTest, TestLockTree, DirectoryFactoryTest, DeleteStatusTest, TestPseudoReturnFields, PathHierarchyTokenizerFactoryTest, TestCloudManagedSchema, TestSQLHandlerNonCloud, ZkCLITest, TestRawTransformer, TestHighlightDedupGrouping, SolrPluginUtilsTest, UniqFieldsUpdateProcessorFactoryTest, CursorPagingTest, BitVectorTest, DistributedDebugComponentTest, TestSolrIndexConfig, TestStressLiveNodes, TestLegacyFieldCache, TimeZoneUtilsTest, TestSchemalessBufferedUpdates, TestFunctionQuery, TestHalfAndHalfDocValues, TestChildDocTransformer, TestAddFieldRealTimeGet, TestTolerantUpdateProcessorRandomCloud, TestReplicationHandler, PKIAuthenticationIntegrationTest, ChaosMonkeySafeLeaderTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionIntegrationTest, ClusterStateUpdateTest, LeaderElectionTest, TestRandomDVFaceting, ShardRoutingCustomTest, TestRecovery, TestHashPartitioner, DistributedSpellCheckComponentTest, TestStressReorder, HardAutoCommitTest, DistributedTermsComponentTest, TestCoreContainer, SpellCheckComponentTest, QueryElevationComponentTest, PeerSyncTest, ConvertedLegacyTest, TestSort, TestLazyCores, TestBadConfig, TestIndexSearcher, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, AnalysisAfterCoreReloadTest, CoreAdminHandlerTest, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTest, TestTrie, SpatialFilterTest, SuggesterWFSTTest, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestUpdate, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, DocValuesMultiTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestWriterPerf, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, TestQueryTypes, TestOmitPositions, FileBasedSpellCheckerTest, TermsComponentTest, RequestHandlersTest, DocumentBuilderTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, FastVectorHighlighterTest, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, IndexSchemaTest, TestJmxIntegration, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, JSONWriterTest, JsonLoaderTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, PingRequestHandlerTest, CSVRequestHandlerTest, TestComponentsName, TestBinaryResponseWriter, TestLFUCache, HighlighterConfigTest, SOLR749Test, TestQuerySenderListener, AlternateDirectoryTest, TestQuerySenderNoQuery, CopyFieldTest, ResponseLogComponentTest, TestStressRecovery, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestSearchPerf, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestPhraseSuggestions, TestCharFilters, EchoParamsTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestJmxMonitoredMap, ScriptEngineTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, PreAnalyzedFieldTest, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, FileUtilsTest, CircularListTest, DistributedMLTComponentTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestSimpleTrackingShardHandler, TestTolerantSearch, TestPathTrie, TestEmbeddedSolrServerConstructors, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, CleanupOldIndexTest, CollectionsAPISolrJTest, DeleteLastCustomShardedReplicaTest, DeleteShardTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, DocValuesNotIndexedTest, LeaderElectionContextKeyTest, LeaderInitiatedRecoveryOnCommitTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerStatusTest, ReplaceNodeTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudPivotFacet, TestConfigSetsAPIZkFailure, TestLeaderInitiatedRecoveryThread, TestLocalFSCloudBackupRestore, TestOnReconnectListenerSupport, TestRebalanceLeaders, TestSSLRandomization, TestSizeLimitedDistributedMap, VMParamsZkACLAndCredentialsProvidersTest, RuleEngineTest, RulesTest]
[junit4] Completed [558/711 (3!)] on J0 in 59.63s, 6 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 43952 lines...]
[JENKINS] Lucene-Solr-6.x-MacOSX (64bit/jdk1.8.0) - Build # 866 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.x-MacOSX/866/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
7 tests failed.
FAILED: org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest
Error Message:
Could not find collection : delLiveColl
Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : delLiveColl
at __randomizedtesting.SeedInfo.seed([12AD24191CFFCBC8:BFCD901201C063BD]:0)
at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:194)
at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:245)
at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:49)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest
Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
at __randomizedtesting.SeedInfo.seed([12AD24191CFFCBC8]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:270)
at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:264)
at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:188)
at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.setupCluster(DistribDocExpirationUpdateProcessorTest.java:58)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1842)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
... 31 more
Caused by: java.io.IOException: Too many open files
at sun.nio.ch.IOUtil.makePipe(Native Method)
at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
at java.nio.channels.Selector.open(Selector.java:227)
at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
at sun.reflect.GeneratedConstructorAccessor22.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1839)
... 37 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest
Error Message:
ObjectTracker found 5 object(s) that were not released!!! [Overseer, ZkController, ZkController, SolrZkClient, SolrZkClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:526) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:754) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) at org.apache.solr.cloud.ZkController.init(ZkController.java:688) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:423) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185) at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299) at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112) at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480) at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244) at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173) at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137) at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873) at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565) at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599) at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285) at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130) at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448) at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306) at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394) at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367) at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386) at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 5 object(s) that were not released!!! [Overseer, ZkController, ZkController, SolrZkClient, SolrZkClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.cloud.Overseer.start(Overseer.java:526)
at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:754)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
at org.apache.solr.cloud.ZkController.init(ZkController.java:688)
at org.apache.solr.cloud.ZkController.<init>(ZkController.java:423)
at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480)
at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244)
at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173)
at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425)
at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480)
at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244)
at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173)
at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.cloud.ZkController.<init>(ZkController.java:425)
at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480)
at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244)
at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173)
at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299)
at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480)
at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244)
at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173)
at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
at org.apache.solr.cloud.ZkController.<init>(ZkController.java:299)
at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
at org.apache.solr.core.CoreContainer.load(CoreContainer.java:480)
at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:244)
at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173)
at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1565)
at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1599)
at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1285)
at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1130)
at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:386)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:249)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([12AD24191CFFCBC8]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
at sun.reflect.GeneratedMethodAccessor64.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest
Error Message:
Could not remove the following files (in the order of attempts): /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001/tempDir-001: java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001/tempDir-001: Too many open files /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001/init-core-data-001: java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001/init-core-data-001: Too many open files /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001: java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001: Too many open files
Stack Trace:
java.io.IOException: Could not remove the following files (in the order of attempts):
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001/tempDir-001: java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001/tempDir-001: Too many open files
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001/init-core-data-001: java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001/init-core-data-001: Too many open files
/Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001: java.nio.file.FileSystemException: /Users/jenkins/workspace/Lucene-Solr-6.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest_12AD24191CFFCBC8-001: Too many open files
at org.apache.lucene.util.IOUtils.rm(IOUtils.java:329)
at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:216)
at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest
Error Message:
39 threads leaked from SUITE scope at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest: 1) Thread[id=18562, name=zkCallback-3999-thread-1, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=18537, name=Scheduler-1885723647, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=18523, name=qtp1303215215-18523-acceptor-0@52d6b77f-ServerConnector@29493f8a{HTTP/1.1,[http/1.1]}{127.0.0.1:53399}, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=18529, name=qtp1479351120-18529-acceptor-0@368e0417-ServerConnector@76e24700{HTTP/1.1,[http/1.1]}{127.0.0.1:53400}, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=18538, name=Scheduler-986507617, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=18519, name=jetty-launcher-3988-thread-1, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=18557, name=jetty-launcher-3988-thread-2-SendThread(127.0.0.1:53397), state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) 8) Thread[id=18533, name=qtp1479351120-18533, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=18563, name=OverseerStateUpdate-97950068419592198-127.0.0.1:53399_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165) at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:183) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=18540, name=org.eclipse.jetty.server.session.HashSessionManager@4ac36f78Timer, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=18561, name=zkCallback-4000-thread-1, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=18521, name=qtp1303215215-18521, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=18511, name=Thread-3506, state=WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1252) at java.lang.Thread.join(Thread.java:1326) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490) 14) Thread[id=18550, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=18549, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=18539, name=org.eclipse.jetty.server.session.HashSessionManager@265c3273Timer, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=18515, name=ProcessThread(sid:0 cport:53397):, state=WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122) 18) Thread[id=18565, name=OverseerHdfsCoreFailoverThread-97950068419592198-127.0.0.1:53399_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=18548, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=18522, name=qtp1479351120-18522, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=18527, name=qtp1303215215-18527, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=18536, name=qtp1479351120-18536, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=18524, name=qtp1303215215-18524, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=18513, name=SessionTracker, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 25) Thread[id=18528, name=qtp1303215215-18528, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=18531, name=qtp1479351120-18531, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=18560, name=jetty-launcher-3988-thread-2-EventThread, state=WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 28) Thread[id=18530, name=qtp1303215215-18530, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=18520, name=jetty-launcher-3988-thread-2, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=18559, name=jetty-launcher-3988-thread-1-EventThread, state=WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 31) Thread[id=18512, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=18534, name=qtp1479351120-18534, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=18535, name=qtp1479351120-18535, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=18532, name=qtp1479351120-18532, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=18525, name=qtp1303215215-18525, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=18526, name=qtp1303215215-18526, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=18558, name=jetty-launcher-3988-thread-1-SendThread(127.0.0.1:53397), state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest] at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method) at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198) at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) 38) Thread[id=18547, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest] at java.lang.Thread.sleep(Native Method) at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=18564, name=OverseerCollectionConfigSetProcessor-97950068419592198-127.0.0.1:53399_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:360) at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 39 threads leaked from SUITE scope at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest:
1) Thread[id=18562, name=zkCallback-3999-thread-1, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=18537, name=Scheduler-1885723647, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=18523, name=qtp1303215215-18523-acceptor-0@52d6b77f-ServerConnector@29493f8a{HTTP/1.1,[http/1.1]}{127.0.0.1:53399}, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=18529, name=qtp1479351120-18529-acceptor-0@368e0417-ServerConnector@76e24700{HTTP/1.1,[http/1.1]}{127.0.0.1:53400}, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:373)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=18538, name=Scheduler-986507617, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=18519, name=jetty-launcher-3988-thread-1, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=18557, name=jetty-launcher-3988-thread-2-SendThread(127.0.0.1:53397), state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
8) Thread[id=18533, name=qtp1479351120-18533, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=18563, name=OverseerStateUpdate-97950068419592198-127.0.0.1:53399_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:183)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=18540, name=org.eclipse.jetty.server.session.HashSessionManager@4ac36f78Timer, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=18561, name=zkCallback-4000-thread-1, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=18521, name=qtp1303215215-18521, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=18511, name=Thread-3506, state=WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at java.lang.Object.wait(Native Method)
at java.lang.Thread.join(Thread.java:1252)
at java.lang.Thread.join(Thread.java:1326)
at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
14) Thread[id=18550, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=18549, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=18539, name=org.eclipse.jetty.server.session.HashSessionManager@265c3273Timer, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=18515, name=ProcessThread(sid:0 cport:53397):, state=WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
18) Thread[id=18565, name=OverseerHdfsCoreFailoverThread-97950068419592198-127.0.0.1:53399_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=18548, name=solr-idle-connections-evictor, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.update.UpdateShardHandler$IdleConnectionsEvictor$1.run(UpdateShardHandler.java:266)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=18522, name=qtp1479351120-18522, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=18527, name=qtp1303215215-18527, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=18536, name=qtp1479351120-18536, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=18524, name=qtp1303215215-18524, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=18513, name=SessionTracker, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at java.lang.Object.wait(Native Method)
at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
25) Thread[id=18528, name=qtp1303215215-18528, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=18531, name=qtp1479351120-18531, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=18560, name=jetty-launcher-3988-thread-2-EventThread, state=WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
28) Thread[id=18530, name=qtp1303215215-18530, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
[...truncated too long message...]
ecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 4) Thread[id=18522, name=qtp1479351120-18522, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.updateSelectedKeys(KQueueSelectorImpl.java:181)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 5) Thread[id=18562, name=zkCallback-3999-thread-1, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 6) Thread[id=18537, name=Scheduler-1885723647, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 7) Thread[id=18538, name=Scheduler-986507617, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 8) Thread[id=18519, name=jetty-launcher-3988-thread-1, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 9) Thread[id=18557, name=jetty-launcher-3988-thread-2-SendThread(127.0.0.1:53397), state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > 10) Thread[id=18521, name=qtp1303215215-18521, state=RUNNABLE, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
[junit4] > at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
[junit4] > at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 11) Thread[id=18558, name=jetty-launcher-3988-thread-1-SendThread(127.0.0.1:53397), state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
[junit4] > 12) Thread[id=18539, name=org.eclipse.jetty.server.session.HashSessionManager@265c3273Timer, state=TIMED_WAITING, group=TGRP-DistribDocExpirationUpdateProcessorTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([12AD24191CFFCBC8]:0)
[junit4] Completed [515/711 (3!)] on J1 in 44.95s, 0 tests, 1 failure, 4 errors <<< FAILURES!
[...truncated 44147 lines...]