You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/17 04:26:54 UTC

[JENKINS] Lucene-Solr-trunk-Windows (64bit/jdk1.8.0-ea-b102) - Build # 3153 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Windows/3153/
Java: 64bit/jdk1.8.0-ea-b102 -XX:+UseCompressedOops -XX:+UseParallelGC

14 tests failed.
REGRESSION:  org.apache.solr.cloud.AliasIntegrationTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.DeleteShardTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest

Error Message:
ERROR: SolrIndexSearcher opens=1 closes=0

Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=1 closes=0
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:327)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:133)
	at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:700)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest

Error Message:
20 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest:     1) Thread[id=3275, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)    2) Thread[id=3286, name=qtp1832944333-3286, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)    3) Thread[id=3272, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:724)    4) Thread[id=3299, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)    5) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:724)    6) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    7) Thread[id=3285, name=qtp1832944333-3285, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)    8) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    9) Thread[id=3287, name=qtp1832944333-3287, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   10) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   11) Thread[id=3293, name=Thread-1339, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:170)         at java.lang.Thread.run(Thread.java:724)   12) Thread[id=3273, name=SessionTracker, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   13) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)   14) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)         at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)   15) Thread[id=3283, name=qtp1832944333-3283, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   16) Thread[id=3284, name=qtp1832944333-3284, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   17) Thread[id=3288, name=qtp1832944333-3288, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)   19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)   20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1224)         at java.lang.Thread.join(Thread.java:1298)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest: 
   1) Thread[id=3275, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   2) Thread[id=3286, name=qtp1832944333-3286, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:724)
   3) Thread[id=3272, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
        at java.lang.Thread.run(Thread.java:724)
   4) Thread[id=3299, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
   5) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:724)
   6) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   7) Thread[id=3285, name=qtp1832944333-3285, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:724)
   8) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   9) Thread[id=3287, name=qtp1832944333-3287, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:724)
  10) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
  11) Thread[id=3293, name=Thread-1339, state=TIMED_WAITING, group=Overseer state updater.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:170)
        at java.lang.Thread.run(Thread.java:724)
  12) Thread[id=3273, name=SessionTracker, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
  13) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
  14) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
        at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:724)
  15) Thread[id=3283, name=qtp1832944333-3283, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:724)
  16) Thread[id=3284, name=qtp1832944333-3284, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:724)
  17) Thread[id=3288, name=qtp1832944333-3288, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:724)
  18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
  19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
        at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:724)
  20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1224)
        at java.lang.Thread.join(Thread.java:1298)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)    3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)    4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:724)    5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)         at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
   3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:724)
   4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:724)
   5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)
        at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
        at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:724)
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)


REGRESSION:  org.apache.solr.cloud.ShardRoutingTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.ShardSplitTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
timed out waiting for collection1 startAt time to exceed: Fri Aug 16 23:14:05 ART 2013

Stack Trace:
java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Fri Aug 16 23:14:05 ART 2013
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:730DC19B7D11763C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1513)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:817)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.schema.TestCloudManagedSchemaAddField.testDistribSearch

Error Message:
Unexpected internal error near index 1 \  ^

Stack Trace:
java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
\
 ^
	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
	at java.util.regex.Pattern.error(Pattern.java:1946)
	at java.util.regex.Pattern.compile(Pattern.java:1693)
	at java.util.regex.Pattern.<init>(Pattern.java:1342)
	at java.util.regex.Pattern.compile(Pattern.java:1028)
	at java.lang.String.split(String.java:2366)
	at java.lang.String.split(String.java:2408)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:491)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBefor

[...truncated too long message...]

kingArrayQueue.java:342)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
   [junit4]    >   18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
   [junit4]    >   19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
   [junit4]    >   20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1224)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1298)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    >    2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:502)
   [junit4]    >         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
   [junit4]    >    3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
   [junit4]    >         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
   [junit4]    >    4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
   [junit4]    >    5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   [junit4]    >    6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   [junit4]    >    7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
   [junit4] Completed in 148.01s, 1 test, 1 failure, 3 errors <<< FAILURES!

[...truncated 480 lines...]
BUILD FAILED
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:389: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:369: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:39: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build.xml:181: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\common-build.xml:450: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:1230: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:873: There were test failures: 321 suites, 1371 tests, 3 suite-level errors, 10 errors, 1 failure, 702 ignored (6 assumptions)

Total time: 49 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0-ea-b102 -XX:+UseCompressedOops -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-trunk-Windows (32bit/jdk1.7.0_25) - Build # 3154 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Windows/3154/
Java: 32bit/jdk1.7.0_25 -client -XX:+UseParallelGC

No tests ran.

Build Log:
[...truncated 31740 lines...]
ERROR: Publisher hudson.tasks.junit.JUnitResultArchiver aborted due to exception
hudson.remoting.RequestAbortedException: hudson.remoting.RequestAbortedException: hudson.remoting.Channel$OrderlyShutdown
	at hudson.remoting.RequestAbortedException.wrapForRethrow(RequestAbortedException.java:41)
	at hudson.remoting.RequestAbortedException.wrapForRethrow(RequestAbortedException.java:34)
	at hudson.remoting.Request.call(Request.java:174)
	at hudson.remoting.Channel.call(Channel.java:714)
	at hudson.FilePath.act(FilePath.java:898)
	at hudson.FilePath.act(FilePath.java:882)
	at hudson.tasks.junit.JUnitParser.parse(JUnitParser.java:87)
	at hudson.tasks.junit.JUnitResultArchiver.parse(JUnitResultArchiver.java:121)
	at hudson.tasks.junit.JUnitResultArchiver.perform(JUnitResultArchiver.java:133)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:19)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:776)
	at hudson.model.Build$BuildExecution.post2(Build.java:183)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:726)
	at hudson.model.Run.execute(Run.java:1622)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:46)
	at hudson.model.ResourceController.execute(ResourceController.java:88)
	at hudson.model.Executor.run(Executor.java:247)
Caused by: hudson.remoting.RequestAbortedException: hudson.remoting.Channel$OrderlyShutdown
	at hudson.remoting.Request.abort(Request.java:299)
	at hudson.remoting.Channel.terminate(Channel.java:774)
	at hudson.remoting.Channel$CloseCommand.execute(Channel.java:919)
	at hudson.remoting.Channel$2.handle(Channel.java:461)
	at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:60)
Caused by: hudson.remoting.Channel$OrderlyShutdown
	... 3 more
Caused by: Command close created at
	at hudson.remoting.Command.<init>(Command.java:56)
	at hudson.remoting.Channel$CloseCommand.<init>(Channel.java:913)
	at hudson.remoting.Channel$CloseCommand.<init>(Channel.java:911)
	at hudson.remoting.Channel.close(Channel.java:994)
	at hudson.remoting.Channel.close(Channel.java:977)
	at hudson.remoting.Channel$CloseCommand.execute(Channel.java:918)
	... 2 more
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-trunk-Windows (64bit/jdk1.8.0-ea-b102) - Build # 3153 - Failure!

Posted by Mark Miller <ma...@gmail.com>.
Ah - I only copied one of two changes I made for Windows over.

- Mark

On Aug 16, 2013, at 11:16 PM, Mark Miller <ma...@gmail.com> wrote:

> This is me - a little surprised because I ran the tests on Windows succesfully, but I'll look at it soon.
> 
> - Mark
> 
> On Aug 16, 2013, at 10:26 PM, Policeman Jenkins Server <je...@thetaphi.de> wrote:
> 
>> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Windows/3153/
>> Java: 64bit/jdk1.8.0-ea-b102 -XX:+UseCompressedOops -XX:+UseParallelGC
>> 
>> 14 tests failed.
>> REGRESSION:  org.apache.solr.cloud.AliasIntegrationTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.cloud.DeleteShardTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest
>> 
>> Error Message:
>> ERROR: SolrIndexSearcher opens=1 closes=0
>> 
>> Stack Trace:
>> java.lang.AssertionError: ERROR: SolrIndexSearcher opens=1 closes=0
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
>> 	at org.junit.Assert.fail(Assert.java:93)
>> 	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:327)
>> 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:133)
>> 	at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:700)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest
>> 
>> Error Message:
>> 20 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest:     1) Thread[id=3275, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)    2) Thread[id=3286, name=qtp1832944333-3286, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)    3) Thread[id=3272, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:724)    4) Thread[id=3299, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)    5) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:724)    6) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    7) Thread[id=3285, name=qtp1832944333-3285, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)    8) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    9) Thread[id=3287, name=qtp1832944333-3287, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   10) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   11) Thread[id=3293, name=Thread-1339, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:170)         at java.lang.Thread.run(Thread.java:724)   12) Thread[id=3273, name=SessionTracker, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   13) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)   14) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)         at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)   15) Thread[id=3283, name=qtp1832944333-3283, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   16) Thread[id=3284, name=qtp1832944333-3284, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   17) Thread[id=3288, name=qtp1832944333-3288, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)   19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)   20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1224)         at java.lang.Thread.join(Thread.java:1298)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)
>> 
>> Stack Trace:
>> com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest: 
>>  1) Thread[id=3275, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>>       at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>>       at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
>>  2) Thread[id=3286, name=qtp1832944333-3286, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>>       at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>>       at java.lang.Thread.run(Thread.java:724)
>>  3) Thread[id=3272, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>>       at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>>       at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>       at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>       at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
>>       at java.lang.Thread.run(Thread.java:724)
>>  4) Thread[id=3299, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>>       at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>>       at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
>>  5) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>>       at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>>       at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>>       at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>>       at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>       at java.lang.Thread.run(Thread.java:724)
>>  6) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>>       at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>>       at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>       at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>       at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>>       at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>>  7) Thread[id=3285, name=qtp1832944333-3285, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>>       at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>>       at java.lang.Thread.run(Thread.java:724)
>>  8) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>>       at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>>       at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>       at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>       at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>>       at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>>  9) Thread[id=3287, name=qtp1832944333-3287, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>>       at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>>       at java.lang.Thread.run(Thread.java:724)
>> 10) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at java.lang.Object.wait(Native Method)
>>       at java.util.TimerThread.mainLoop(Timer.java:552)
>>       at java.util.TimerThread.run(Timer.java:505)
>> 11) Thread[id=3293, name=Thread-1339, state=TIMED_WAITING, group=Overseer state updater.]
>>       at java.lang.Thread.sleep(Native Method)
>>       at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:170)
>>       at java.lang.Thread.run(Thread.java:724)
>> 12) Thread[id=3273, name=SessionTracker, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at java.lang.Object.wait(Native Method)
>>       at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
>> 13) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
>>       at java.lang.Object.wait(Native Method)
>>       at java.lang.Object.wait(Object.java:502)
>>       at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
>> 14) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
>>       at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
>>       at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
>>       at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>       at java.lang.Thread.run(Thread.java:724)
>> 15) Thread[id=3283, name=qtp1832944333-3283, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>>       at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>>       at java.lang.Thread.run(Thread.java:724)
>> 16) Thread[id=3284, name=qtp1832944333-3284, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>>       at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>>       at java.lang.Thread.run(Thread.java:724)
>> 17) Thread[id=3288, name=qtp1832944333-3288, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>>       at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>>       at java.lang.Thread.run(Thread.java:724)
>> 18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>>       at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>>       at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
>> 19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>>       at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>>       at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>       at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>       at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
>>       at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>       at java.lang.Thread.run(Thread.java:724)
>> 20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]
>>       at java.lang.Object.wait(Native Method)
>>       at java.lang.Thread.join(Thread.java:1224)
>>       at java.lang.Thread.join(Thread.java:1298)
>>       at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
>>       at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)
>>       at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
>> 
>> 
>> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest
>> 
>> Error Message:
>> There are still zombie threads that couldn't be terminated:    1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)    3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)    4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:724)    5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)         at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)
>> 
>> Stack Trace:
>> com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
>>  1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>       at java.lang.Object.wait(Native Method)
>>       at java.util.TimerThread.mainLoop(Timer.java:552)
>>       at java.util.TimerThread.run(Timer.java:505)
>>  2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
>>       at java.lang.Object.wait(Native Method)
>>       at java.lang.Object.wait(Object.java:502)
>>       at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
>>  3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>       at java.lang.Thread.run(Thread.java:724)
>>  4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
>>       at sun.misc.Unsafe.park(Native Method)
>>       at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>>       at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>>       at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>>       at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>>       at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>>       at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>       at java.lang.Thread.run(Thread.java:724)
>>  5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
>>       at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
>>       at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
>>       at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>       at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>       at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>>       at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>>  6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)
>>       at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)
>>       at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)
>>       at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
>>       at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
>>       at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)
>>       at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>       at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>       at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>>       at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>>  7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>>       at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>>       at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>>       at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>       at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>       at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
>>       at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>       at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>       at java.lang.Thread.run(Thread.java:724)
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
>> 
>> 
>> REGRESSION:  org.apache.solr.cloud.ShardRoutingTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.cloud.ShardSplitTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication
>> 
>> Error Message:
>> timed out waiting for collection1 startAt time to exceed: Fri Aug 16 23:14:05 ART 2013
>> 
>> Stack Trace:
>> java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Fri Aug 16 23:14:05 ART 2013
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:730DC19B7D11763C]:0)
>> 	at org.junit.Assert.fail(Assert.java:93)
>> 	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1513)
>> 	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:817)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:724)
>> 
>> 
>> REGRESSION:  org.apache.solr.schema.TestCloudManagedSchemaAddField.testDistribSearch
>> 
>> Error Message:
>> Unexpected internal error near index 1 \  ^
>> 
>> Stack Trace:
>> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
>> \
>> ^
>> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
>> 	at java.util.regex.Pattern.error(Pattern.java:1946)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
>> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
>> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
>> 	at java.lang.String.split(String.java:2366)
>> 	at java.lang.String.split(String.java:2408)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
>> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:491)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBefor
>> 
>> [...truncated too long message...]
>> 
>> kingArrayQueue.java:342)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>>  [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>>  [junit4]    >   18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at sun.misc.Unsafe.park(Native Method)
>>  [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>>  [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>>  [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>>  [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
>>  [junit4]    >   19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>>  [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>  [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>  [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
>>  [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>  [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>>  [junit4]    >   20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at java.lang.Object.wait(Native Method)
>>  [junit4]    >         at java.lang.Thread.join(Thread.java:1224)
>>  [junit4]    >         at java.lang.Thread.join(Thread.java:1298)
>>  [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
>>  [junit4]    >         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)
>>  [junit4]    >         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)
>>  [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
>>  [junit4]    >    1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at java.lang.Object.wait(Native Method)
>>  [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:552)
>>  [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
>>  [junit4]    >    2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at java.lang.Object.wait(Native Method)
>>  [junit4]    >         at java.lang.Object.wait(Object.java:502)
>>  [junit4]    >         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
>>  [junit4]    >    3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>  [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>>  [junit4]    >    4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at sun.misc.Unsafe.park(Native Method)
>>  [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>>  [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>>  [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>>  [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>>  [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>>  [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>  [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>>  [junit4]    >    5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
>>  [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>  [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>  [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>>  [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>>  [junit4]    >    6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)
>>  [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
>>  [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)
>>  [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>  [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>  [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>>  [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>>  [junit4]    >    7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>>  [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>>  [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>>  [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>>  [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
>>  [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>  [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>  [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>>  [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
>>  [junit4] Completed in 148.01s, 1 test, 1 failure, 3 errors <<< FAILURES!
>> 
>> [...truncated 480 lines...]
>> BUILD FAILED
>> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:389: The following error occurred while executing this line:
>> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:369: The following error occurred while executing this line:
>> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:39: The following error occurred while executing this line:
>> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build.xml:181: The following error occurred while executing this line:
>> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\common-build.xml:450: The following error occurred while executing this line:
>> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:1230: The following error occurred while executing this line:
>> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:873: There were test failures: 321 suites, 1371 tests, 3 suite-level errors, 10 errors, 1 failure, 702 ignored (6 assumptions)
>> 
>> Total time: 49 minutes 38 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Description set: Java: 64bit/jdk1.8.0-ea-b102 -XX:+UseCompressedOops -XX:+UseParallelGC
>> Archiving artifacts
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>> 
> 


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-trunk-Windows (64bit/jdk1.8.0-ea-b102) - Build # 3153 - Failure!

Posted by Mark Miller <ma...@gmail.com>.
This is me - a little surprised because I ran the tests on Windows succesfully, but I'll look at it soon.

- Mark

On Aug 16, 2013, at 10:26 PM, Policeman Jenkins Server <je...@thetaphi.de> wrote:

> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Windows/3153/
> Java: 64bit/jdk1.8.0-ea-b102 -XX:+UseCompressedOops -XX:+UseParallelGC
> 
> 14 tests failed.
> REGRESSION:  org.apache.solr.cloud.AliasIntegrationTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.cloud.DeleteShardTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest
> 
> Error Message:
> ERROR: SolrIndexSearcher opens=1 closes=0
> 
> Stack Trace:
> java.lang.AssertionError: ERROR: SolrIndexSearcher opens=1 closes=0
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
> 	at org.junit.Assert.fail(Assert.java:93)
> 	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:327)
> 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:133)
> 	at sun.reflect.GeneratedMethodAccessor25.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:700)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest
> 
> Error Message:
> 20 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest:     1) Thread[id=3275, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)    2) Thread[id=3286, name=qtp1832944333-3286, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)    3) Thread[id=3272, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:724)    4) Thread[id=3299, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)    5) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:724)    6) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    7) Thread[id=3285, name=qtp1832944333-3285, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)    8) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    9) Thread[id=3287, name=qtp1832944333-3287, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   10) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)   11) Thread[id=3293, name=Thread-1339, state=TIMED_WAITING, group=Overseer state updater.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:170)         at java.lang.Thread.run(Thread.java:724)   12) Thread[id=3273, name=SessionTracker, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   13) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)   14) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)         at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)   15) Thread[id=3283, name=qtp1832944333-3283, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   16) Thread[id=3284, name=qtp1832944333-3284, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   17) Thread[id=3288, name=qtp1832944333-3288, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:724)   18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)   19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)   20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1224)         at java.lang.Thread.join(Thread.java:1298)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.cloud.RecoveryZkTest: 
>   1) Thread[id=3275, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
>   2) Thread[id=3286, name=qtp1832944333-3286, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>        at java.lang.Thread.run(Thread.java:724)
>   3) Thread[id=3272, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
>        at java.lang.Thread.run(Thread.java:724)
>   4) Thread[id=3299, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
>   5) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        at java.lang.Thread.run(Thread.java:724)
>   6) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>   7) Thread[id=3285, name=qtp1832944333-3285, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>        at java.lang.Thread.run(Thread.java:724)
>   8) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>   9) Thread[id=3287, name=qtp1832944333-3287, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>        at java.lang.Thread.run(Thread.java:724)
>  10) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at java.lang.Object.wait(Native Method)
>        at java.util.TimerThread.mainLoop(Timer.java:552)
>        at java.util.TimerThread.run(Timer.java:505)
>  11) Thread[id=3293, name=Thread-1339, state=TIMED_WAITING, group=Overseer state updater.]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:170)
>        at java.lang.Thread.run(Thread.java:724)
>  12) Thread[id=3273, name=SessionTracker, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at java.lang.Object.wait(Native Method)
>        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
>  13) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
>        at java.lang.Object.wait(Native Method)
>        at java.lang.Object.wait(Object.java:502)
>        at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
>  14) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
>        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
>        at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
>        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>        at java.lang.Thread.run(Thread.java:724)
>  15) Thread[id=3283, name=qtp1832944333-3283, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>        at java.lang.Thread.run(Thread.java:724)
>  16) Thread[id=3284, name=qtp1832944333-3284, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>        at java.lang.Thread.run(Thread.java:724)
>  17) Thread[id=3288, name=qtp1832944333-3288, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2076)
>        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>        at java.lang.Thread.run(Thread.java:724)
>  18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
>  19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>        at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
>        at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>        at java.lang.Thread.run(Thread.java:724)
>  20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]
>        at java.lang.Object.wait(Native Method)
>        at java.lang.Thread.join(Thread.java:1224)
>        at java.lang.Thread.join(Thread.java:1298)
>        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
>        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)
>        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.RecoveryZkTest
> 
> Error Message:
> There are still zombie threads that couldn't be terminated:    1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)    3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)    4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:724)    5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)         at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)    7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:724)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
>   1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>        at java.lang.Object.wait(Native Method)
>        at java.util.TimerThread.mainLoop(Timer.java:552)
>        at java.util.TimerThread.run(Timer.java:505)
>   2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
>        at java.lang.Object.wait(Native Method)
>        at java.lang.Object.wait(Object.java:502)
>        at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
>   3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>        at java.lang.Thread.run(Thread.java:724)
>   4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        at java.lang.Thread.run(Thread.java:724)
>   5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
>        at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
>        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
>        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>   6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)
>        at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)
>        at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)
>        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
>        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
>        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)
>        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>   7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>        at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
>        at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>        at java.lang.Thread.run(Thread.java:724)
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
> 
> 
> REGRESSION:  org.apache.solr.cloud.ShardRoutingTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.cloud.ShardSplitTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication
> 
> Error Message:
> timed out waiting for collection1 startAt time to exceed: Fri Aug 16 23:14:05 ART 2013
> 
> Stack Trace:
> java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Fri Aug 16 23:14:05 ART 2013
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:730DC19B7D11763C]:0)
> 	at org.junit.Assert.fail(Assert.java:93)
> 	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1513)
> 	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:817)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:724)
> 
> 
> REGRESSION:  org.apache.solr.schema.TestCloudManagedSchemaAddField.testDistribSearch
> 
> Error Message:
> Unexpected internal error near index 1 \  ^
> 
> Stack Trace:
> java.util.regex.PatternSyntaxException: Unexpected internal error near index 1
> \
> ^
> 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F:29404F450F667FB3]:0)
> 	at java.util.regex.Pattern.error(Pattern.java:1946)
> 	at java.util.regex.Pattern.compile(Pattern.java:1693)
> 	at java.util.regex.Pattern.<init>(Pattern.java:1342)
> 	at java.util.regex.Pattern.compile(Pattern.java:1028)
> 	at java.lang.String.split(String.java:2366)
> 	at java.lang.String.split(String.java:2408)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.getRelativeSolrHomePath(AbstractFullDistribZkTestBase.java:485)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:463)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:348)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:294)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:833)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:491)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBefor
> 
> [...truncated too long message...]
> 
> kingArrayQueue.java:342)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
>   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>   [junit4]    >   18) Thread[id=3292, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-EventThread, state=WAITING, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
>   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
>   [junit4]    >   19) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
>   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>   [junit4]    >   20) Thread[id=3271, name=Thread-1329, state=WAITING, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at java.lang.Object.wait(Native Method)
>   [junit4]    >         at java.lang.Thread.join(Thread.java:1224)
>   [junit4]    >         at java.lang.Thread.join(Thread.java:1298)
>   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
>   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:107)
>   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:221)
>   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
>   [junit4]    >    1) Thread[id=3280, name=HashSessionScavenger-83, state=TIMED_WAITING, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at java.lang.Object.wait(Native Method)
>   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:552)
>   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
>   [junit4]    >    2) Thread[id=3297, name=Thread-1340, state=WAITING, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at java.lang.Object.wait(Native Method)
>   [junit4]    >         at java.lang.Object.wait(Object.java:502)
>   [junit4]    >         at org.apache.solr.core.CloserThread.run(CoreContainer.java:961)
>   [junit4]    >    3) Thread[id=3282, name=qtp1832944333-3282 Acceptor0 SelectChannelConnector@127.0.0.1:57260, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:956)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>   [junit4]    >    4) Thread[id=3296, name=searcherExecutor-1934-thread-1, state=WAITING, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
>   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2037)
>   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>   [junit4]    >    5) Thread[id=3291, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
>   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>   [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>   [junit4]    >    6) Thread[id=3298, name=TEST-RecoveryZkTest.testDistribSearch-seed#[A8A6C15D78391F8F]-SendThread(127.0.0.1:57253), state=RUNNABLE, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)
>   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
>   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)
>   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>   [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
>   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
>   [junit4]    >    7) Thread[id=3281, name=qtp1832944333-3281 Selector0, state=RUNNABLE, group=TGRP-RecoveryZkTest]
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
>   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
>   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
>   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
>   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
>   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>   [junit4]    >         at java.lang.Thread.run(Thread.java:724)
>   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A8A6C15D78391F8F]:0)
>   [junit4] Completed in 148.01s, 1 test, 1 failure, 3 errors <<< FAILURES!
> 
> [...truncated 480 lines...]
> BUILD FAILED
> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:389: The following error occurred while executing this line:
> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:369: The following error occurred while executing this line:
> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:39: The following error occurred while executing this line:
> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build.xml:181: The following error occurred while executing this line:
> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\common-build.xml:450: The following error occurred while executing this line:
> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:1230: The following error occurred while executing this line:
> C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:873: There were test failures: 321 suites, 1371 tests, 3 suite-level errors, 10 errors, 1 failure, 702 ignored (6 assumptions)
> 
> Total time: 49 minutes 38 seconds
> Build step 'Invoke Ant' marked build as failure
> Description set: Java: 64bit/jdk1.8.0-ea-b102 -XX:+UseCompressedOops -XX:+UseParallelGC
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
> 


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org