You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/01/06 21:26:23 UTC

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1061 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1061/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.test

Error Message:
timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([C470F112EA5E5039:4C24CEC844A23DC1]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
	at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
	at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
	at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics

Error Message:
minorMerge: 3 expected:<4> but was:<3>

Stack Trace:
java.lang.AssertionError: minorMerge: 3 expected:<4> but was:<3>
	at __randomizedtesting.SeedInfo.seed([C470F112EA5E5039:8A0CCAE2AD0AB02]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics(SolrIndexMetricsTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10997 lines...]
   [junit4] Suite: org.apache.solr.update.SolrIndexMetricsTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.update.SolrIndexMetricsTest_C470F112EA5E5039-001/init-core-data-001
   [junit4]   2> 450381 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 450381 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 450382 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 450402 WARN  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 450402 WARN  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 450402 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 450411 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 450490 WARN  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 450492 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 450503 WARN  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 450504 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 450504 WARN  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 450508 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 450508 WARN  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 450508 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 450509 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 450527 WARN  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 450527 WARN  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 450528 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 450536 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 450612 WARN  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 450613 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 450632 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 450633 WARN  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 450633 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 450633 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.update.SolrIndexMetricsTest_C470F112EA5E5039-001/init-core-data-001/]
   [junit4]   2> 450660 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 450660 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 450663 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@405b53e8[collection1] main]
   [junit4]   2> 450663 WARN  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 450663 INFO  (coreLoadExecutor-1516-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 450663 INFO  (searcherExecutor-1517-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@405b53e8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 450664 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 450667 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testIndexMetrics
   [junit4]   2> 451285 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 451285 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@24e94d2
   [junit4]   2> 451322 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d9926fa[collection1] main]
   [junit4]   2> 451322 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 451322 INFO  (searcherExecutor-1517-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1d9926fa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C100) Uninverting(_1(7.0.0):C100) Uninverting(_2(7.0.0):C100) Uninverting(_3(7.0.0):C100) Uninverting(_5(7.0.0):C100) Uninverting(_6(7.0.0):C100) Uninverting(_7(7.0.0):C100) Uninverting(_9(7.0.0):C100) Uninverting(_a(7.0.0):C100) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 451323 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 451349 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 451349 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 451349 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 451358 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 451442 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 451444 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 451452 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 451452 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 451461 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 451461 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.m.SolrMetricManager Replacing existing reporter 'default' in registry 'solr.core.collection1': [org.apache.solr.metrics.reporters.SolrJmxReporter@61cdc398: domain = solr.core.collection1, service url = null, agent id = null]
   [junit4]   2> 451465 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.update.SolrIndexMetricsTest_C470F112EA5E5039-001/init-core-data-001/]
   [junit4]   2> 451477 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 451477 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 451478 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@727a8f71[collection1] main]
   [junit4]   2> 451478 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 451478 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 451479 INFO  (searcherExecutor-1521-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@727a8f71[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C100) Uninverting(_1(7.0.0):C100) Uninverting(_2(7.0.0):C100) Uninverting(_3(7.0.0):C100) Uninverting(_5(7.0.0):C100) Uninverting(_6(7.0.0):C100) Uninverting(_7(7.0.0):C100) Uninverting(_9(7.0.0):C100) Uninverting(_a(7.0.0):C100) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 451856 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 451945 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@32046d71[collection1] main]
   [junit4]   2> 451946 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3838461d
   [junit4]   2> 451946 INFO  (searcherExecutor-1521-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@32046d71[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(7.0.0):C300) Uninverting(_8(7.0.0):C300) Uninverting(_c(7.0.0):C300) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 451946 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 451950 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[C470F112EA5E5039]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testIndexMetrics
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrIndexMetricsTest -Dtests.method=testIndexMetrics -Dtests.seed=C470F112EA5E5039 -Dtests.slow=true -Dtests.locale=zh-HK -Dtests.timezone=America/Moncton -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 1.29s J1 | SolrIndexMetricsTest.testIndexMetrics <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: minorMerge: 3 expected:<4> but was:<3>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C470F112EA5E5039:8A0CCAE2AD0AB02]:0)
   [junit4]    > 	at org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics(SolrIndexMetricsTest.java:70)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 451953 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 451953 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    x:collection1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=554588197
   [junit4]   2> 451953 INFO  (coreCloseExecutor-1525-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@69e1440c
   [junit4]   2> 451956 INFO  (coreCloseExecutor-1525-thread-1-processing-x:collection1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 451956 INFO  (SUITE-SolrIndexMetricsTest-seed#[C470F112EA5E5039]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.update.SolrIndexMetricsTest_C470F112EA5E5039-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {range_facet_l_dv=PostingsFormat(name=Direct), multiDefault=FST50, intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), foo_s=FST50, id=PostingsFormat(name=Direct), range_facet_i_dv=FST50, range_facet_l=FST50, timestamp=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Memory), multiDefault=DocValuesFormat(name=Asserting), intDefault=DocValuesFormat(name=Lucene70), range_facet_i_dv=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Memory), foo_s=DocValuesFormat(name=Asserting), timestamp=DocValuesFormat(name=Asserting), range_facet_l=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=615, maxMBSortInHeap=5.725459981148864, sim=RandomSimilarity(queryNorm=true): {}, locale=zh-HK, timezone=America/Moncton
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=176481080,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestFoldingMultitermQuery, EnumFieldTest, LukeRequestHandlerTest, TestExceedMaxTermLength, IndexSchemaTest, TestManagedSynonymFilterFactory, CdcrVersionReplicationTest, BigEndianAscendingWordSerializerTest, TestFilteredDocIdSet, TestRandomCollapseQParserPlugin, ResourceLoaderTest, TestDistributedStatsComponentCardinality, TestFreeTextSuggestions, TestReRankQParserPlugin, TestTrieFacet, TestImpersonationWithHadoopAuth, RequestLoggingTest, ExternalFileFieldSortTest, TestPHPSerializedResponseWriter, TestRebalanceLeaders, TestFieldResource, TestNonDefinedSimilarityFactory, JavabinLoaderTest, UpdateParamsTest, SolrCoreCheckLockOnStartupTest, TestCodecSupport, MultiThreadedOCPTest, CollectionsAPISolrJTest, ZkStateReaderTest, UpdateRequestProcessorFactoryTest, TestAuthenticationFramework, SolrIndexSplitterTest, TestSolrCloudWithSecureImpersonation, AnalysisErrorHandlingTest, BasicAuthIntegrationTest, TestObjectReleaseTracker, TestUtils, HdfsRecoverLeaseTest, TestStressRecovery, TestLMDirichletSimilarityFactory, SharedFSAutoReplicaFailoverTest, BasicDistributedZk2Test, SolrMetricReporterTest, TestShardHandlerFactory, CloneFieldUpdateProcessorFactoryTest, TestOrdValues, TestFieldCacheSanityChecker, TestXmlQParserPlugin, BlockJoinFacetRandomTest, CdcrReplicationHandlerTest, TestIndexingPerformance, DebugComponentTest, CircularListTest, DateMathParserTest, TestRandomFlRTGCloud, NoCacheHeaderTest, SortSpecParsingTest, SolrIndexMetricsTest]
   [junit4] Completed [117/675 (1!)] on J1 in 1.59s, 1 test, 1 failure <<< FAILURES!

[...truncated 143 lines...]
   [junit4] Suite: org.apache.solr.cloud.PeerSyncReplicationTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/init-core-data-001
   [junit4]   2> 493623 INFO  (SUITE-PeerSyncReplicationTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 493623 INFO  (SUITE-PeerSyncReplicationTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 493625 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 493625 INFO  (Thread-919) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 493625 INFO  (Thread-919) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 493725 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ZkTestServer start zk server on port:34308
   [junit4]   2> 493741 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 493744 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 493747 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 493750 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 493752 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 493754 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 493756 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 493759 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 493761 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 493763 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 493765 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractZkTestCase put /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 493768 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x159757444c80001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 493910 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/control-001/cores/collection1
   [junit4]   2> 493912 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 493913 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66bc417d{/,null,AVAILABLE}
   [junit4]   2> 493918 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3f8ac02c{HTTP/1.1,[http/1.1]}{127.0.0.1:48510}
   [junit4]   2> 493918 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.Server Started @497048ms
   [junit4]   2> 493918 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/tempDir-001/control/data, hostContext=/, hostPort=48510, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/control-001/cores}
   [junit4]   2> 493919 ERROR (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 493919 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 493919 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 493919 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 493919 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-06T20:24:31.469Z
   [junit4]   2> 493923 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 493923 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/control-001/solr.xml
   [junit4]   2> 493930 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34308/solr
   [junit4]   2> 493967 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:48510_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:48510_
   [junit4]   2> 493968 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:48510_    ] o.a.s.c.Overseer Overseer (id=97238009195790340-127.0.0.1:48510_-n_0000000000) starting
   [junit4]   2> 493980 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:48510_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48510_
   [junit4]   2> 493983 INFO  (OverseerStateUpdate-97238009195790340-127.0.0.1:48510_-n_0000000000) [n:127.0.0.1:48510_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 494036 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:48510_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/control-001/cores
   [junit4]   2> 494036 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:48510_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 494042 INFO  (OverseerStateUpdate-97238009195790340-127.0.0.1:48510_-n_0000000000) [n:127.0.0.1:48510_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 495055 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 495067 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 495146 WARN  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 495148 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 495165 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 495165 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/control-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/control-001/cores/collection1/data/]
   [junit4]   2> 495166 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 495167 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6628213440410655]
   [junit4]   2> 495173 WARN  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 495182 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 495182 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 495183 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 495183 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 495184 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=44, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4515890212616338]
   [junit4]   2> 495185 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@50fdaa84[collection1] main]
   [junit4]   2> 495187 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 495187 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 495187 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 495188 INFO  (searcherExecutor-1844-thread-1-processing-n:127.0.0.1:48510_ x:collection1 c:control_collection) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@50fdaa84[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 495188 INFO  (coreLoadExecutor-1843-thread-1-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_ c:control_collection   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555808148770521088
   [junit4]   2> 495198 INFO  (coreZkRegister-1836-thread-1-processing-n:127.0.0.1:48510_ x:collection1 c:control_collection) [n:127.0.0.1:48510_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 495198 INFO  (coreZkRegister-1836-thread-1-processing-n:127.0.0.1:48510_ x:collection1 c:control_collection) [n:127.0.0.1:48510_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 495198 INFO  (coreZkRegister-1836-thread-1-processing-n:127.0.0.1:48510_ x:collection1 c:control_collection) [n:127.0.0.1:48510_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48510/collection1/
   [junit4]   2> 495198 INFO  (coreZkRegister-1836-thread-1-processing-n:127.0.0.1:48510_ x:collection1 c:control_collection) [n:127.0.0.1:48510_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 495198 INFO  (coreZkRegister-1836-thread-1-processing-n:127.0.0.1:48510_ x:collection1 c:control_collection) [n:127.0.0.1:48510_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:48510/collection1/ has no replicas
   [junit4]   2> 495206 INFO  (coreZkRegister-1836-thread-1-processing-n:127.0.0.1:48510_ x:collection1 c:control_collection) [n:127.0.0.1:48510_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48510/collection1/ shard1
   [junit4]   2> 495216 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 495217 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34308/solr ready
   [junit4]   2> 495217 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 495217 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
   [junit4]   2> 495257 INFO  (coreZkRegister-1836-thread-1-processing-n:127.0.0.1:48510_ x:collection1 c:control_collection) [n:127.0.0.1:48510_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 495369 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-1-001/cores/collection1
   [junit4]   2> 495370 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-1-001
   [junit4]   2> 495371 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 495373 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6c9839c2{/,null,AVAILABLE}
   [junit4]   2> 495373 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@76ac7561{HTTP/1.1,[http/1.1]}{127.0.0.1:63252}
   [junit4]   2> 495373 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.Server Started @498504ms
   [junit4]   2> 495373 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=63252, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-1-001/cores}
   [junit4]   2> 495374 ERROR (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 495374 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 495374 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 495374 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 495374 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-06T20:24:32.924Z
   [junit4]   2> 495378 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 495378 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-1-001/solr.xml
   [junit4]   2> 495385 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34308/solr
   [junit4]   2> 495400 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 495407 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:63252_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63252_
   [junit4]   2> 495410 INFO  (zkCallback-634-thread-4-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 495412 INFO  (zkCallback-638-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 495413 INFO  (zkCallback-644-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 495565 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:63252_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-1-001/cores
   [junit4]   2> 495565 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:63252_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 495568 INFO  (OverseerStateUpdate-97238009195790340-127.0.0.1:48510_-n_0000000000) [n:127.0.0.1:48510_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 495672 INFO  (zkCallback-644-thread-2-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 496582 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 496610 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 496677 WARN  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 496710 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 496726 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 496727 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-1-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-1-001/cores/collection1/data/]
   [junit4]   2> 496727 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 496729 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6628213440410655]
   [junit4]   2> 496735 WARN  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 496747 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 496747 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 496749 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 496749 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 496750 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=44, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4515890212616338]
   [junit4]   2> 496751 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7652e0ac[collection1] main]
   [junit4]   2> 496753 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 496753 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 496753 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 496754 INFO  (coreLoadExecutor-1854-thread-1-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555808150412591104
   [junit4]   2> 496777 INFO  (coreZkRegister-1849-thread-1-processing-n:127.0.0.1:63252_ x:collection1 c:collection1) [n:127.0.0.1:63252_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 496778 INFO  (coreZkRegister-1849-thread-1-processing-n:127.0.0.1:63252_ x:collection1 c:collection1) [n:127.0.0.1:63252_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 496778 INFO  (coreZkRegister-1849-thread-1-processing-n:127.0.0.1:63252_ x:collection1 c:collection1) [n:127.0.0.1:63252_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:63252/collection1/
   [junit4]   2> 496778 INFO  (coreZkRegister-1849-thread-1-processing-n:127.0.0.1:63252_ x:collection1 c:collection1) [n:127.0.0.1:63252_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 496778 INFO  (coreZkRegister-1849-thread-1-processing-n:127.0.0.1:63252_ x:collection1 c:collection1) [n:127.0.0.1:63252_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:63252/collection1/ has no replicas
   [junit4]   2> 496782 INFO  (searcherExecutor-1855-thread-1-processing-n:127.0.0.1:63252_ x:collection1 c:collection1) [n:127.0.0.1:63252_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7652e0ac[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 496784 INFO  (coreZkRegister-1849-thread-1-processing-n:127.0.0.1:63252_ x:collection1 c:collection1) [n:127.0.0.1:63252_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:63252/collection1/ shard1
   [junit4]   2> 496891 INFO  (zkCallback-644-thread-2-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 496941 INFO  (coreZkRegister-1849-thread-1-processing-n:127.0.0.1:63252_ x:collection1 c:collection1) [n:127.0.0.1:63252_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 497047 INFO  (zkCallback-644-thread-2-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 497392 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-2-001/cores/collection1
   [junit4]   2> 497392 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-2-001
   [junit4]   2> 497393 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 497395 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@71e65a91{/,null,AVAILABLE}
   [junit4]   2> 497395 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@433c1403{HTTP/1.1,[http/1.1]}{127.0.0.1:62307}
   [junit4]   2> 497395 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.Server Started @500527ms
   [junit4]   2> 497395 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=62307, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-2-001/cores}
   [junit4]   2> 497395 ERROR (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 497397 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 497397 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 497397 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 497397 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-06T20:24:34.947Z
   [junit4]   2> 497404 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 497404 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-2-001/solr.xml
   [junit4]   2> 497412 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34308/solr
   [junit4]   2> 497427 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:62307_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 497432 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:62307_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62307_
   [junit4]   2> 497437 INFO  (zkCallback-634-thread-3-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 497438 INFO  (zkCallback-644-thread-2-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 497438 INFO  (zkCallback-638-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 497450 INFO  (zkCallback-650-thread-2-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 497536 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:62307_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-2-001/cores
   [junit4]   2> 497536 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:62307_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 497539 INFO  (OverseerStateUpdate-97238009195790340-127.0.0.1:48510_-n_0000000000) [n:127.0.0.1:48510_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 497642 INFO  (zkCallback-644-thread-2-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 497644 INFO  (zkCallback-650-thread-2-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 498575 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 498593 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 498678 WARN  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 498679 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 498711 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 498712 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-2-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/../../../../../../../../../../export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-2-001/cores/collection1/data/]
   [junit4]   2> 498712 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 498714 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6628213440410655]
   [junit4]   2> 498720 WARN  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 498731 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 498731 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 498733 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 498733 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 498735 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=44, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4515890212616338]
   [junit4]   2> 498735 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2327f563[collection1] main]
   [junit4]   2> 498737 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 498737 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 498737 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 498738 INFO  (searcherExecutor-1866-thread-1-processing-n:127.0.0.1:62307_ x:collection1 c:collection1) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2327f563[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 498738 INFO  (coreLoadExecutor-1865-thread-1-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555808152492965888
   [junit4]   2> 498743 INFO  (coreZkRegister-1860-thread-1-processing-n:127.0.0.1:62307_ x:collection1 c:collection1) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 498743 INFO  (updateExecutor-647-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 498744 INFO  (recoveryExecutor-648-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 498745 INFO  (recoveryExecutor-648-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 498745 INFO  (recoveryExecutor-648-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1]
   [junit4]   2> 498745 INFO  (recoveryExecutor-648-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 498745 INFO  (recoveryExecutor-648-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Publishing state of core [collection1] as recovering, leader is [http://127.0.0.1:63252/collection1/] and I am [http://127.0.0.1:62307/collection1/]
   [junit4]   2> 498747 INFO  (recoveryExecutor-648-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:63252]; [WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1:62307_&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 498755 INFO  (qtp441244970-4076) [n:127.0.0.1:63252_    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 498759 INFO  (qtp441244970-4076) [n:127.0.0.1:63252_    ] o.a.s.h.a.PrepRecoveryOp Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 498759 INFO  (qtp441244970-4076) [n:127.0.0.1:63252_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:62307_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:62307","node_name":"127.0.0.1:62307_","state":"down"}
   [junit4]   2> 498861 INFO  (zkCallback-644-thread-2-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 498861 INFO  (zkCallback-650-thread-2-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 499357 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-3-001/cores/collection1
   [junit4]   2> 499358 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-3-001
   [junit4]   2> 499359 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 499361 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@192cfc15{/,null,AVAILABLE}
   [junit4]   2> 499361 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7b111609{HTTP/1.1,[http/1.1]}{127.0.0.1:41400}
   [junit4]   2> 499361 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.Server Started @502494ms
   [junit4]   2> 499361 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=41400, coreRootDirectory=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-3-001/cores}
   [junit4]   2> 499361 ERROR (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 499364 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 499364 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 499364 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 499364 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-06T20:24:36.914Z
   [junit4]   2> 499368 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 499368 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-3-001/solr.xml
   [junit4]   2> 499375 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34308/solr
   [junit4]   2> 499388 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:41400_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 499395 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:41400_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41400_
   [junit4]   2> 499399 INFO  (zkCallback-638-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 499400 INFO  (zkCallback-650-thread-2-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 499399 INFO  (zkCallback-644-thread-2-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 499399 INFO  (zkCallback-634-thread-2-processing-n:127.0.0.1:48510_) [n:127.0.0.1:48510_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 499401 INFO  (zkCallback-657-thread-2-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 499458 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:41400_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-3-001/cores
   [junit4]   2> 499459 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [n:127.0.0.1:41400_    ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
   [junit4]   2> 499462 INFO  (OverseerStateUpdate-97238009195790340-127.0.0.1:48510_-n_0000000000) [n:127.0.0.1:48510_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 499565 INFO  (zkCallback-644-thread-2-processing-n:127.0.0.1:63252_) [n:127.0.0.1:63252_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 499566 INFO  (zkCallback-650-thread-2-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 499566 INFO  (zkCallback-657-thread-2-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 499760 INFO  (qtp441244970-4076) [n:127.0.0.1:63252_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:62307_, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"core":"collection1","base_url":"http://127.0.0.1:62307","node_name":"127.0.0.1:62307_","state":"recovering"}
   [junit4]   2> 499760 INFO  (qtp441244970-4076) [n:127.0.0.1:63252_    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 499760 INFO  (qtp441244970-4076) [n:127.0.0.1:63252_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:62307_&onlyIfLeaderActive=true&core=collection1&coreNodeName=core_node2&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1006
   [junit4]   2> 500483 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 500497 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 500590 WARN  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 500592 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 500609 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 500609 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-3-001/cores/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001/shard-3-001/cores/collection1/data/]
   [junit4]   2> 500609 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3781c1ab
   [junit4]   2> 500611 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6628213440410655]
   [junit4]   2> 500618 WARN  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 500630 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 500630 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=1000 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 500632 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 500632 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 500632 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=44, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4515890212616338]
   [junit4]   2> 500633 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1ff95f0[collection1] main]
   [junit4]   2> 500635 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 500635 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 500635 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 500636 INFO  (searcherExecutor-1877-thread-1-processing-n:127.0.0.1:41400_ x:collection1 c:collection1) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1ff95f0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 500637 INFO  (coreLoadExecutor-1876-thread-1-processing-n:127.0.0.1:41400_) [n:127.0.0.1:41400_ c:collection1   x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555808154484211712
   [junit4]   2> 500642 INFO  (coreZkRegister-1871-thread-1-processing-n:127.0.0.1:41400_ x:collection1 c:collection1) [n:127.0.0.1:41400_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.c.ZkController Core needs to recover:collection1
   [junit4]   2> 500643 INFO  (updateExecutor-654-thread-1-processing-n:127.0.0.1:41400_ x:collection1 s:shard1 c:collection1 r:core_node3) [n:127.0.0.1:41400_ c:collection1 s:shard1 r:core_node3 x:collection1] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 500643 INFO  (recoveryExecutor-655-thread-1-processing-n:127.0.0.1:

[...truncated too long message...]

470F112EA5E5039]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6c9839c2{/,null,UNAVAILABLE}
   [junit4]   2> 697436 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ChaosMonkey monkey: stop shard! 62307
   [junit4]   2> 697436 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1771948178
   [junit4]   2> 697437 WARN  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
   [junit4]   2> 697442 WARN  (updateExecutor-661-thread-2-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.u.DefaultSolrCoreState Skipping recovery because Solr is shutdown
   [junit4]   2> 697554 INFO  (zkCallback-664-thread-4-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 700932 INFO  (zkCallback-664-thread-2-processing-n:127.0.0.1:62307_) [n:127.0.0.1:62307_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 701216 INFO  (recoveryExecutor-662-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy RecoveryStrategy has been closed
   [junit4]   2> 701216 INFO  (recoveryExecutor-662-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[false]
   [junit4]   2> 701216 INFO  (recoveryExecutor-662-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4914847d
   [junit4]   2> 701216 WARN  (recoveryExecutor-662-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.RecoveryStrategy Stopping recovery for core=[collection1] coreNodeName=[core_node2]
   [junit4]   2> 701228 INFO  (recoveryExecutor-662-thread-1-processing-n:127.0.0.1:62307_ x:collection1 s:shard1 c:collection1 r:core_node2) [n:127.0.0.1:62307_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 701229 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.Overseer Overseer (id=97238009195790354-127.0.0.1:62307_-n_0000000004) closing
   [junit4]   2> 701230 INFO  (OverseerStateUpdate-97238009195790354-127.0.0.1:62307_-n_0000000004) [n:127.0.0.1:62307_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:62307_
   [junit4]   2> 701233 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 701235 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@6212b5ce{HTTP/1.1,[http/1.1]}{127.0.0.1:62307}
   [junit4]   2> 701235 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3a0fa34{/,null,UNAVAILABLE}
   [junit4]   2> 701236 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ChaosMonkey monkey: stop shard! 41400
   [junit4]   2> 701239 INFO  (TEST-PeerSyncReplicationTest.test-seed#[C470F112EA5E5039]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34308 34308
   [junit4]   2> 701268 INFO  (Thread-919) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34308 34308
   [junit4]   2> 701270 WARN  (Thread-919) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	5	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 	2	/solr/collections/collection1/leader_elect/shard1/election/97238009195790345-core_node1-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/97238009195790345-127.0.0.1:63252_-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	207	/solr/overseer/collection-queue-work
   [junit4]   2> 	43	/solr/overseer/queue
   [junit4]   2> 	15	/solr/overseer/queue-work
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	5	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=PeerSyncReplicationTest -Dtests.method=test -Dtests.seed=C470F112EA5E5039 -Dtests.slow=true -Dtests.locale=bg-BG -Dtests.timezone=America/Noronha -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  208s J1 | PeerSyncReplicationTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: timeout waiting to see all nodes active
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C470F112EA5E5039:4C24CEC844A23DC1]:0)
   [junit4]    > 	at org.apache.solr.cloud.PeerSyncReplicationTest.waitTillNodesActive(PeerSyncReplicationTest.java:311)
   [junit4]    > 	at org.apache.solr.cloud.PeerSyncReplicationTest.bringUpDeadNodeAndEnsureNoReplication(PeerSyncReplicationTest.java:262)
   [junit4]    > 	at org.apache.solr.cloud.PeerSyncReplicationTest.forceNodeFailureAndDoPeerSync(PeerSyncReplicationTest.java:244)
   [junit4]    > 	at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:133)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 701272 INFO  (SUITE-PeerSyncReplicationTest-seed#[C470F112EA5E5039]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.PeerSyncReplicationTest_C470F112EA5E5039-001
   [junit4]   2> Jan 06, 2017 8:27:58 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {other_tl1=FST50, range_facet_l_dv=Lucene50(blocksize=128), rnd_s=FSTOrd50, multiDefault=FSTOrd50, intDefault=FST50, a_i1=FST50, range_facet_l=FST50, _version_=FST50, a_t=FST50, id=Lucene50(blocksize=128), range_facet_i_dv=FST50, text=PostingsFormat(name=LuceneVarGapDocFreqInterval), timestamp=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Memory), range_facet_i_dv=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1211, maxMBSortInHeap=5.378027162390759, sim=RandomSimilarity(queryNorm=true): {}, locale=bg-BG, timezone=America/Noronha
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=261860960,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestFoldingMultitermQuery, EnumFieldTest, LukeRequestHandlerTest, TestExceedMaxTermLength, IndexSchemaTest, TestManagedSynonymFilterFactory, CdcrVersionReplicationTest, BigEndianAscendingWordSerializerTest, TestFilteredDocIdSet, TestRandomCollapseQParserPlugin, ResourceLoaderTest, TestDistributedStatsComponentCardinality, TestFreeTextSuggestions, TestReRankQParserPlugin, TestTrieFacet, TestImpersonationWithHadoopAuth, RequestLoggingTest, ExternalFileFieldSortTest, TestPHPSerializedResponseWriter, TestRebalanceLeaders, TestFieldResource, TestNonDefinedSimilarityFactory, JavabinLoaderTest, UpdateParamsTest, SolrCoreCheckLockOnStartupTest, TestCodecSupport, MultiThreadedOCPTest, CollectionsAPISolrJTest, ZkStateReaderTest, UpdateRequestProcessorFactoryTest, TestAuthenticationFramework, SolrIndexSplitterTest, TestSolrCloudWithSecureImpersonation, AnalysisErrorHandlingTest, BasicAuthIntegrationTest, TestObjectReleaseTracker, TestUtils, HdfsRecoverLeaseTest, TestStressRecovery, TestLMDirichletSimilarityFactory, SharedFSAutoReplicaFailoverTest, BasicDistributedZk2Test, SolrMetricReporterTest, TestShardHandlerFactory, CloneFieldUpdateProcessorFactoryTest, TestOrdValues, TestFieldCacheSanityChecker, TestXmlQParserPlugin, BlockJoinFacetRandomTest, CdcrReplicationHandlerTest, TestIndexingPerformance, DebugComponentTest, CircularListTest, DateMathParserTest, TestRandomFlRTGCloud, NoCacheHeaderTest, SortSpecParsingTest, SolrIndexMetricsTest, TestSolr4Spatial2, WrapperMergePolicyFactoryTest, BooleanFieldTest, TestSystemIdResolver, TestHashQParserPlugin, TestConfigSetsAPI, TestDeleteCollectionOnDownNodes, DistributedFacetPivotSmallAdvancedTest, TestConfigReload, TestSha256AuthenticationProvider, PeerSyncReplicationTest]
   [junit4] Completed [164/675 (2!)] on J1 in 207.92s, 1 test, 1 failure <<< FAILURES!

[...truncated 64110 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1065 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1065/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenRenew

Error Message:
expected:<200> but was:<403>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<403>
	at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11:8AB134D6220492B5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.renewDelegationToken(TestSolrCloudWithDelegationTokens.java:130)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.verifyDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:315)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:332)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsNNFailoverTest.test

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:63593 within 45000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:63593 within 45000 ms
	at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11:357EFF12B43422E9]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:106)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:86)
	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:80)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.distribSetUp(AbstractDistribZkTestBase.java:81)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribSetUp(AbstractFullDistribZkTestBase.java:219)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:955)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:63593 within 45000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
	... 38 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
5 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest:     1) Thread[id=1610, name=SyncThread:0, state=WAITING, group=TGRP-HdfsNNFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)    2) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=1607, name=Thread-359, state=WAITING, group=TGRP-HdfsNNFailoverTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1249)         at java.lang.Thread.join(Thread.java:1323)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)    4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-HdfsNNFailoverTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)    5) Thread[id=1609, name=SessionTracker, state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   1) Thread[id=1610, name=SyncThread:0, state=WAITING, group=TGRP-HdfsNNFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
   2) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=1607, name=Thread-359, state=WAITING, group=TGRP-HdfsNNFailoverTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1249)
        at java.lang.Thread.join(Thread.java:1323)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-HdfsNNFailoverTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   5) Thread[id=1609, name=SessionTracker, state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
	at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsNNFailoverTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
        at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
        at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11]:0)




Build Log:
[...truncated 10826 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001/init-core-data-001
   [junit4]   2> 172675 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 172675 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001/tempDir-001
   [junit4]   2> 172676 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 172676 INFO  (Thread-343) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 172676 INFO  (Thread-343) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 172776 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:42837
   [junit4]   2> 172793 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 172793 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 172798 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@69ada8b8{/solr,null,AVAILABLE}
   [junit4]   2> 172800 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@57e4c34c{/solr,null,AVAILABLE}
   [junit4]   2> 172804 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@65df4f79{HTTP/1.1,[http/1.1]}{127.0.0.1:40654}
   [junit4]   2> 172804 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.Server Started @175515ms
   [junit4]   2> 172804 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40654}
   [junit4]   2> 172805 ERROR (jetty-launcher-296-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-08T14:22:58.814Z
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3e5ad244{HTTP/1.1,[http/1.1]}{127.0.0.1:51468}
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.Server Started @175516ms
   [junit4]   2> 172805 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51468}
   [junit4]   2> 172805 ERROR (jetty-launcher-296-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 172806 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.0.0
   [junit4]   2> 172806 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 172806 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 172806 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-08T14:22:58.815Z
   [junit4]   2> 172812 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 172812 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 172824 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42837/solr
   [junit4]   2> 172826 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:42837/solr
   [junit4]   2> 172837 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x1597e75fc810004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 172862 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:51468_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51468_solr
   [junit4]   2> 172863 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:51468_solr    ] o.a.s.c.Overseer Overseer (id=97247912177762309-127.0.0.1:51468_solr-n_0000000000) starting
   [junit4]   2> 172879 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:51468_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51468_solr
   [junit4]   2> 172882 INFO  (zkCallback-307-thread-1-processing-n:127.0.0.1:40654_solr) [n:127.0.0.1:40654_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 172918 INFO  (OverseerStateUpdate-97247912177762309-127.0.0.1:51468_solr-n_0000000000) [n:127.0.0.1:51468_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 172919 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:40654_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40654_solr
   [junit4]   2> 172922 INFO  (zkCallback-307-thread-1-processing-n:127.0.0.1:40654_solr) [n:127.0.0.1:40654_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 172937 INFO  (zkCallback-306-thread-1-processing-n:127.0.0.1:51468_solr) [n:127.0.0.1:51468_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 173180 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:51468_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 173180 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:51468_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 173180 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:51468_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 173182 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:40654_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 173182 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:40654_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 173182 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:40654_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 173196 INFO  (jetty-launcher-296-thread-2-EventThread) [n:127.0.0.1:51468_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 173197 INFO  (jetty-launcher-296-thread-1-EventThread) [n:127.0.0.1:40654_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 173267 INFO  (jetty-launcher-296-thread-2) [n:127.0.0.1:51468_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001/tempDir-001/node1/.
   [junit4]   2> 173274 INFO  (jetty-launcher-296-thread-1) [n:127.0.0.1:40654_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001/tempDir-001/node2/.
   [junit4]   2> 173471 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 173558 ERROR (qtp710559443-1609) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_1]
   [junit4]   2> 173576 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 173577 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42837/solr ready
   [junit4]   2> 173587 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173594 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 173597 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42837/solr ready
   [junit4]   2> 173600 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173630 ERROR (qtp450497154-1614) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_2]
   [junit4]   2> 173638 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173646 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173646 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancel-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 173675 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 173684 WARN  (qtp710559443-1606) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 173685 WARN  (qtp710559443-1606) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 173685 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173687 WARN  (qtp450497154-1601) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 173688 WARN  (qtp450497154-1601) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 173688 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173696 WARN  (qtp710559443-1602) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 173697 WARN  (qtp710559443-1602) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 173697 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173698 WARN  (qtp450497154-1601) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 173699 WARN  (qtp450497154-1601) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 173699 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173699 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 173752 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 173757 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 173759 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42837/solr ready
   [junit4]   2> 173762 INFO  (qtp450497154-1601) [n:127.0.0.1:40654_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 173762 INFO  (qtp450497154-1601) [n:127.0.0.1:40654_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 173762 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1483921379770&s=Fr4VIeIcylBaJOkGhsRiRh2PBaA="; Path=/; Domain=127.0.0.1; Expires= , 09-1 -2017 00:22:59 GMT; HttpOnly". Invalid 'expires' attribute: , 09-1 -2017 00:22:59 GMT
   [junit4]   2> 173765 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationOtherOp-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 173796 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 173812 INFO  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgDeigFZooKE3gUEFBlkp1Smd8e9FLg3L5sQZbSrOZIRB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 173812 INFO  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgDeigFZooKE3gUEFBlkp1Smd8e9FLg3L5sQZbSrOZIRB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 173819 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 173820 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:42837/solr ready
   [junit4]   2> 173824 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173830 INFO  (qtp450497154-1601) [n:127.0.0.1:40654_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgDeigFZooKE3gUEFBlkp1Smd8e9FLg3L5sQZbSrOZIRB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 173830 INFO  (qtp450497154-1601) [n:127.0.0.1:40654_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgDeigFZooKE3gUEFBlkp1Smd8e9FLg3L5sQZbSrOZIRB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 173836 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173837 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenVerify-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 173875 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 173886 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 173895 INFO  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgEtigFZooKFLQYEFFbqAZ5kWo5jtshoBIcVb-NDr22XB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 173895 INFO  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVl-dgEtigFZooKFLQYEFFbqAZ5kWo5jtshoBIcVb-NDr22XB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 173901 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenSolrClient-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 173924 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 173950 ERROR (qtp450497154-1603) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_7]
   [junit4]   2> 173954 ERROR (qtp710559443-1608) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_7]
   [junit4]   2> 173967 ERROR (qtp450497154-1617) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_8]
   [junit4]   2> 173973 ERROR (qtp710559443-1607) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_8]
   [junit4]   2> 173974 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> 174000 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 174015 WARN  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar is trying to renew a token with wrong password
   [junit4]   2> 174015 WARN  (qtp710559443-1609) [n:127.0.0.1:51468_solr    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 174017 WARN  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[BD2AC0C81AC84F11]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Path=/; Domain=127.0.0.1; Expires= , 01-1 -1970 00:00:00 GMT; HttpOnly". Invalid 'expires' attribute: , 01-1 -1970 00:00:00 GMT
   [junit4]   2> 174017 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrCloudWithDelegationTokens -Dtests.method=testDelegationTokenRenew -Dtests.seed=BD2AC0C81AC84F11 -Dtests.slow=true -Dtests.locale=ko-KR -Dtests.timezone=America/Indiana/Knox -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.05s J1 | TestSolrCloudWithDelegationTokens.testDelegationTokenRenew <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<403>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11:8AB134D6220492B5]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.renewDelegationToken(TestSolrCloudWithDelegationTokens.java:130)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.verifyDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:315)
   [junit4]    > 	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:332)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 174057 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 174083 INFO  (TEST-TestSolrCloudWithDelegationTokens.testZNodePaths-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 174084 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@65df4f79{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 174085 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1771798306
   [junit4]   2> 174086 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@3e5ad244{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 174086 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=53018233
   [junit4]   2> 174092 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.c.Overseer Overseer (id=97247912177762309-127.0.0.1:51468_solr-n_0000000000) closing
   [junit4]   2> 174093 INFO  (OverseerStateUpdate-97247912177762309-127.0.0.1:51468_solr-n_0000000000) [n:127.0.0.1:51468_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:51468_solr
   [junit4]   2> 175595 WARN  (zkCallback-307-thread-1-processing-n:127.0.0.1:40654_solr) [n:127.0.0.1:40654_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 175595 INFO  (jetty-launcher-296-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 175595 WARN  (zkCallback-306-thread-1-processing-n:127.0.0.1:51468_solr) [n:127.0.0.1:51468_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 175595 ERROR (Thread[Thread-363,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:40654_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 175595 INFO  (jetty-launcher-296-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 175596 ERROR (Thread[Thread-360,5,TGRP-TestSolrCloudWithDelegationTokens]) [n:127.0.0.1:51468_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 175602 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x1597e75fc810008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 175602 INFO  (jetty-launcher-296-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@57e4c34c{/solr,null,UNAVAILABLE}
   [junit4]   2> 175605 INFO  (jetty-launcher-296-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@69ada8b8{/solr,null,UNAVAILABLE}
   [junit4]   2> 175609 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42837 42837
   [junit4]   2> 175631 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxnFactory Ignoring exception
   [junit4]   2> java.nio.channels.ClosedChannelException
   [junit4]   2> 	at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:235)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:188)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 175632 INFO  (Thread-343) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:42837 42837
   [junit4]   2> 175634 WARN  (Thread-343) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 175636 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.TestSolrCloudWithDelegationTokens_BD2AC0C81AC84F11-001
   [junit4]   2> Jan 08, 2017 2:23:01 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=428, maxMBSortInHeap=5.553593308142943, sim=RandomSimilarity(queryNorm=true): {}, locale=ko-KR, timezone=America/Indiana/Knox
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=40833776,total=197660672
   [junit4]   2> NOTE: All tests run in this JVM: [TestRuleBasedAuthorizationPlugin, NoCacheHeaderTest, TestFieldCacheReopen, SolrCoreMetricManagerTest, TestConfigSetsAPI, LeaderElectionIntegrationTest, PreAnalyzedFieldTest, TestSolrCloudWithSecureImpersonation, TestAuthenticationFramework, BadIndexSchemaTest, TestComponentsName, ResponseHeaderTest, AsyncCallRequestStatusResponseTest, TestSortByMinMaxFunction, SpellPossibilityIteratorTest, TestSchemaResource, TestReplicationHandlerBackup, InfoHandlerTest, ClusterStateTest, ZkSolrClientTest, TestSimpleTrackingShardHandler, TestTestInjection, TestXmlQParser, TestInitQParser, HdfsBasicDistributedZk2Test, TestFastWriter, TestJmxMonitoredMap, TemplateUpdateProcessorTest, SolrMetricReporterTest, TestSolr4Spatial, TestNoOpRegenerator, TestSolrCloudWithDelegationTokens]
   [junit4] Completed [57/676 (1!)] on J1 in 3.49s, 8 tests, 1 failure <<< FAILURES!

[...truncated 55 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsNNFailoverTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/init-core-data-001
   [junit4]   2> 214030 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 214030 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 214626 WARN  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 215649 WARN  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 215959 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
   [junit4]   2> 215989 WARN  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 216080 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 216137 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_solaris.vm_36490_hdfs____.hffxba/webapp
   [junit4]   2> 217800 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:36490
   [junit4]   2> 218874 WARN  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 218896 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 218910 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_64144_datanode____.3siw12/webapp
   [junit4]   2> 219593 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:64144
   [junit4]   2> 220092 WARN  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 220095 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 220112 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Extract jar:file:/export/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_57301_datanode____.oo31xj/webapp
   [junit4]   2> 221313 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:57301
   [junit4]   2> 222105 INFO  (IPC Server handler 5 on 50684) [    ] BlockStateChange BLOCK* processReport: from storage DS-973dbff0-4810-4d3f-a6ad-7ec37ebcaa56 node DatanodeRegistration(127.0.0.1:46433, datanodeUuid=24fd6232-c359-471d-a0f0-d76dbc930d2e, infoPort=33548, infoSecurePort=0, ipcPort=63593, storageInfo=lv=-56;cid=testClusterID;nsid=1525593412;c=0), blocks: 0, hasStaleStorage: true, processing time: 4 msecs
   [junit4]   2> 222106 INFO  (IPC Server handler 2 on 50684) [    ] BlockStateChange BLOCK* processReport: from storage DS-e6a44d72-20a5-4d62-a9f6-91051665d38a node DatanodeRegistration(127.0.0.1:38002, datanodeUuid=aa35785a-ab9f-4c33-874c-3f53750b86a3, infoPort=52061, infoSecurePort=0, ipcPort=61050, storageInfo=lv=-56;cid=testClusterID;nsid=1525593412;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 222106 INFO  (IPC Server handler 5 on 50684) [    ] BlockStateChange BLOCK* processReport: from storage DS-3a588cb3-d006-42ec-9aee-924ab9d4c30a node DatanodeRegistration(127.0.0.1:46433, datanodeUuid=24fd6232-c359-471d-a0f0-d76dbc930d2e, infoPort=33548, infoSecurePort=0, ipcPort=63593, storageInfo=lv=-56;cid=testClusterID;nsid=1525593412;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 222129 INFO  (IPC Server handler 2 on 50684) [    ] BlockStateChange BLOCK* processReport: from storage DS-7009a6f9-00ed-42ac-97c2-4accfec4317d node DatanodeRegistration(127.0.0.1:38002, datanodeUuid=aa35785a-ab9f-4c33-874c-3f53750b86a3, infoPort=52061, infoSecurePort=0, ipcPort=61050, storageInfo=lv=-56;cid=testClusterID;nsid=1525593412;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 222906 INFO  (TEST-HdfsNNFailoverTest.test-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 222906 INFO  (Thread-359) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 222906 INFO  (Thread-359) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 223006 INFO  (TEST-HdfsNNFailoverTest.test-seed#[BD2AC0C81AC84F11]) [    ] o.a.s.c.ZkTestServer start zk server on port:63593
   [junit4]   2> 223009 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:41363 got version 0 expected version 9
   [junit4]   2> 224372 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:60391 got version 0 expected version 9
   [junit4]   2> 226111 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:44827 got version 0 expected version 9
   [junit4]   2> 228205 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:40233 got version 0 expected version 9
   [junit4]   2> 229929 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:63452 got version 0 expected version 9
   [junit4]   2> 231070 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:45557 got version 0 expected version 9
   [junit4]   2> 232555 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:36093 got version 0 expected version 9
   [junit4]   2> 233766 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:63200 got version 0 expected version 9
   [junit4]   2> 235394 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:57636 got version 0 expected version 9
   [junit4]   2> 237244 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:35493 got version 0 expected version 9
   [junit4]   2> 238370 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:38704 got version 0 expected version 9
   [junit4]   2> 240023 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:35108 got version 0 expected version 9
   [junit4]   2> 241809 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:44341 got version 0 expected version 9
   [junit4]   2> 243883 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:48548 got version 0 expected version 9
   [junit4]   2> 245735 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:45283 got version 0 expected version 9
   [junit4]   2> 246862 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:57530 got version 0 expected version 9
   [junit4]   2> 248482 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:43575 got version 0 expected version 9
   [junit4]   2> 250435 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:62368 got version 0 expected version 9
   [junit4]   2> 251862 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:52824 got version 0 expected version 9
   [junit4]   2> 253555 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:51417 got version 0 expected version 9
   [junit4]   2> 255295 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:52686 got version 0 expected version 9
   [junit4]   2> 257095 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:62613 got version 0 expected version 9
   [junit4]   2> 258815 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:43324 got version 0 expected version 9
   [junit4]   2> 260569 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:34597 got version 0 expected version 9
   [junit4]   2> 262557 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:58015 got version 0 expected version 9
   [junit4]   2> 264268 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:49334 got version 0 expected version 9
   [junit4]   2> 266357 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:64259 got version 0 expected version 9
   [junit4]   2> 267627 WARN  (Socket Reader #1 for port 63593) [    ] o.a.h.i.Server Incorrect header or version mismatch from 127.0.0.1:44373 got version 0 expected version 9
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsNNFailoverTest -Dtests.method=test -Dtests.seed=BD2AC0C81AC84F11 -Dtests.slow=true -Dtests.locale=ar-TN -Dtests.timezone=America/Guadeloupe -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   45.6s J0 | HdfsNNFailoverTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:63593 within 45000 ms
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11:357EFF12B43422E9]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:106)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:86)
   [junit4]    > 	at org.apache.solr.cloud.AbstractZkTestCase.buildZooKeeper(AbstractZkTestCase.java:80)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.distribSetUp(AbstractDistribZkTestBase.java:81)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribSetUp(AbstractFullDistribZkTestBase.java:219)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:955)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:63593 within 45000 ms
   [junit4]    > 	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:233)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
   [junit4]    > 	... 38 more
   [junit4]   2> 268498 WARN  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 268508 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 268614 WARN  (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to solaris-vm/127.0.0.1:50684) [    ] o.a.h.h.s.d.DataNode BPOfferService for Block pool BP-221125253-127.0.0.1-1483885425298 (Datanode Uuid 24fd6232-c359-471d-a0f0-d76dbc930d2e) service to solaris-vm/127.0.0.1:50684 interrupted
   [junit4]   2> 268614 WARN  (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data3/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data4/]]  heartbeating to solaris-vm/127.0.0.1:50684) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-221125253-127.0.0.1-1483885425298 (Datanode Uuid 24fd6232-c359-471d-a0f0-d76dbc930d2e) service to solaris-vm/127.0.0.1:50684
   [junit4]   2> 268621 WARN  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
   [junit4]   2> 268634 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0
   [junit4]   2> 268641 WARN  (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to solaris-vm/127.0.0.1:50684) [    ] o.a.h.h.s.d.DataNode BPOfferService for Block pool BP-221125253-127.0.0.1-1483885425298 (Datanode Uuid aa35785a-ab9f-4c33-874c-3f53750b86a3) service to solaris-vm/127.0.0.1:50684 interrupted
   [junit4]   2> 268641 WARN  (DataNode: [[[DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data1/, [DISK]file:/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001/tempDir-001/hdfsBaseDir/data/data2/]]  heartbeating to solaris-vm/127.0.0.1:50684) [    ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-221125253-127.0.0.1-1483885425298 (Datanode Uuid aa35785a-ab9f-4c33-874c-3f53750b86a3) service to solaris-vm/127.0.0.1:50684
   [junit4]   2> 268961 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.m.log Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@solaris-vm:0
   [junit4]   2> 269072 ERROR (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.h.m.l.MethodMetric Error invoking method getBlocksTotal
   [junit4]   2> java.lang.reflect.InvocationTargetException
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest.teardownClass(HdfsNNFailoverTest.java:48)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 269078 INFO  (SUITE-HdfsNNFailoverTest-seed#[BD2AC0C81AC84F11]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsNNFailoverTest_BD2AC0C81AC84F11-001
   [junit4]   2> Jan 08, 2017 2:24:39 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 29 leaked thread(s).
   [junit4]   2> Jan 08, 2017 2:24:59 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 5 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   [junit4]   2>    1) Thread[id=1610, name=SyncThread:0, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
   [junit4]   2>    2) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]   2>         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2>         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    3) Thread[id=1607, name=Thread-359, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
   [junit4]   2>         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]   2>         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]   2>    4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   [junit4]   2>    5) Thread[id=1609, name=SessionTracker, state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]   2> Jan 08, 2017 2:24:59 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=1610, name=SyncThread:0, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    2) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    3) Thread[id=1607, name=Thread-359, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>    5) Thread[id=1609, name=SessionTracker, state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2> 289152 ERROR (SyncThread:0) [    ] o.a.z.s.SyncRequestProcessor Severe unrecoverable error, exiting
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
   [junit4]   2> 289153 ERROR (SessionTracker) [    ] o.a.z.s.SessionTrackerImpl Unexpected interruption
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]   2> 289153 ERROR (ProcessThread(sid:0 cport:-1):) [    ] o.a.z.s.PrepRequestProcessor Unexpected interruption
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   [junit4]   2> 289157 WARN  (Thread-359) [    ] o.a.s.c.ZkTestServer Server interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.lang.Object.wait(Native Method)
   [junit4]   2> 	at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2> 	at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
   [junit4]   2> 	at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]   2> 	at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]   2> Jan 08, 2017 2:25:02 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
   [junit4]   2>         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]   2>         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]   2>         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]   2>         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]   2>         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]   2>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]   2>         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1492, maxMBSortInHeap=6.367441842716158, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-TN, timezone=America/Guadeloupe
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=3,free=84815800,total=254988288
   [junit4]   2> NOTE: All tests run in this JVM: [TestObjectReleaseTracker, TestBackupRepositoryFactory, TestBinaryField, PreAnalyzedUpdateProcessorTest, TestSolrXml, TestXmlQParserPlugin, TestSolrCloudWithKerberosAlt, RecoveryZkTest, CursorPagingTest, TestIBSimilarityFactory, SolrXmlInZkTest, SortSpecParsingTest, VersionInfoTest, SolrIndexConfigTest, TestJmxIntegration, HttpSolrCallGetCoreTest, TestSort, TestRemoteStreaming, QueryElevationComponentTest, TestUpdate, SimplePostToolTest, TestSolrCLIRunExample, TestLazyCores, TestLeaderElectionWithEmptyReplica, MetricsHandlerTest, TestCustomStream, HdfsNNFailoverTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsNNFailoverTest -Dtests.seed=BD2AC0C81AC84F11 -Dtests.slow=true -Dtests.locale=ar-TN -Dtests.timezone=America/Guadeloupe -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | HdfsNNFailoverTest (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsNNFailoverTest: 
   [junit4]    >    1) Thread[id=1610, name=SyncThread:0, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:127)
   [junit4]    >    2) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    3) Thread[id=1607, name=Thread-359, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1249)
   [junit4]    >         at java.lang.Thread.join(Thread.java:1323)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
   [junit4]    >         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
   [junit4]    >    4) Thread[id=1611, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
   [junit4]    >    5) Thread[id=1609, name=SessionTracker, state=TIMED_WAITING, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=1608, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-HdfsNNFailoverTest]
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollArrayWrapper.interrupt(DevPollArrayWrapper.java:300)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.wakeup(DevPollSelectorImpl.java:207)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BD2AC0C81AC84F11]:0)
   [junit4] Completed [77/676 (2!)] on J0 in 78.20s, 1 test, 3 errors <<< FAILURES!

[...truncated 64357 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1064 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1064/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  org.apache.solr.update.AutoCommitTest.testMaxDocs

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([1FBDFA3E765036C4:A63C2CE15ABA324E]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:818)
	at org.apache.solr.update.AutoCommitTest.testMaxDocs(AutoCommitTest.java:225)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=id:14&qt=standard&start=0&rows=20&version=2.2
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:811)
	... 40 more


FAILED:  org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics

Error Message:
minorMerge: 3 expected:<4> but was:<3>

Stack Trace:
java.lang.AssertionError: minorMerge: 3 expected:<4> but was:<3>
	at __randomizedtesting.SeedInfo.seed([1FBDFA3E765036C4:D36DC782B6DECDFF]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics(SolrIndexMetricsTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([1FBDFA3E765036C4:7702CF14A6CA2428]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:280)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11666 lines...]
   [junit4] Suite: org.apache.solr.update.SolrIndexMetricsTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SolrIndexMetricsTest_1FBDFA3E765036C4-001/init-core-data-001
   [junit4]   2> 1115809 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1115809 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 1115810 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1115837 WARN  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1115837 WARN  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 1115837 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1115845 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 1115933 WARN  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1115936 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1115946 WARN  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1115949 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1115949 WARN  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1115952 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1115952 WARN  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1115952 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1115953 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1115975 WARN  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1115975 WARN  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 1115975 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1115983 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1116066 WARN  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1116068 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1116076 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1116077 WARN  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1116077 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1116077 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SolrIndexMetricsTest_1FBDFA3E765036C4-001/init-core-data-001/]
   [junit4]   2> 1116100 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1116100 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1116103 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f1b8aa8[collection1] main]
   [junit4]   2> 1116103 WARN  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 1116103 INFO  (coreLoadExecutor-3915-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1116103 INFO  (searcherExecutor-3916-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3f1b8aa8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1116104 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 1116107 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testIndexMetrics
   [junit4]   2> 1117087 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1117088 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@7e2d25da
   [junit4]   2> 1117127 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@761c1afd[collection1] main]
   [junit4]   2> 1117128 INFO  (searcherExecutor-3916-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@761c1afd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):C100) Uninverting(_1(7.0.0):C100) Uninverting(_2(7.0.0):C100) Uninverting(_3(7.0.0):C100) Uninverting(_5(7.0.0):C100) Uninverting(_6(7.0.0):C100) Uninverting(_7(7.0.0):C100) Uninverting(_9(7.0.0):C100) Uninverting(_a(7.0.0):C100) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 1117128 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1117128 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1117159 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1117159 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 1117159 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1117168 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1117291 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1117293 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1117302 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1117302 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1117386 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1117430 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.SolrMetricManager Replacing existing reporter 'default' in registry 'solr.core.collection1': [org.apache.solr.metrics.reporters.SolrJmxReporter@10bc84a2: domain = solr.core.collection1, service url = null, agent id = null]
   [junit4]   2> 1117434 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SolrIndexMetricsTest_1FBDFA3E765036C4-001/init-core-data-001/]
   [junit4]   2> 1117449 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1117449 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1117487 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@16c389fb[collection1] main]
   [junit4]   2> 1117487 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 1117487 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1117488 INFO  (searcherExecutor-3920-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@16c389fb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(7.0.0):C300) Uninverting(_3(7.0.0):C100) Uninverting(_5(7.0.0):C100) Uninverting(_6(7.0.0):C100) Uninverting(_7(7.0.0):C100) Uninverting(_9(7.0.0):C100) Uninverting(_a(7.0.0):C100) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 1117990 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1118073 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1267ff6f[collection1] main]
   [junit4]   2> 1118073 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1ab30082
   [junit4]   2> 1118073 INFO  (searcherExecutor-3920-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1267ff6f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(7.0.0):C300) Uninverting(_8(7.0.0):C300) Uninverting(_c(7.0.0):C300) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 1118074 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1118078 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testIndexMetrics
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrIndexMetricsTest -Dtests.method=testIndexMetrics -Dtests.seed=1FBDFA3E765036C4 -Dtests.slow=true -Dtests.locale=ja-JP -Dtests.timezone=US/Mountain -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 1.98s J0 | SolrIndexMetricsTest.testIndexMetrics <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: minorMerge: 3 expected:<4> but was:<3>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1FBDFA3E765036C4:D36DC782B6DECDFF]:0)
   [junit4]    > 	at org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics(SolrIndexMetricsTest.java:70)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1118080 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 1118080 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    x:collection1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=98901261
   [junit4]   2> 1118081 INFO  (coreCloseExecutor-3924-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7b0a26f4
   [junit4]   2> 1118081 INFO  (coreCloseExecutor-3924-thread-1-processing-x:collection1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1118082 INFO  (SUITE-SolrIndexMetricsTest-seed#[1FBDFA3E765036C4]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SolrIndexMetricsTest_1FBDFA3E765036C4-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {range_facet_l_dv=FSTOrd50, multiDefault=FST50, intDefault=PostingsFormat(name=MockRandom), foo_s=FST50, id=FSTOrd50, range_facet_i_dv=FST50, range_facet_l=FST50, timestamp=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), multiDefault=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Lucene70), id=DocValuesFormat(name=Asserting), foo_s=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Lucene70), range_facet_l=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=985, maxMBSortInHeap=7.344751155979984, sim=RandomSimilarity(queryNorm=true): {}, locale=ja-JP, timezone=US/Mountain
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=180678424,total=438304768
   [junit4]   2> NOTE: All tests run in this JVM: [SystemInfoHandlerTest, TestDynamicFieldCollectionResource, HighlighterMaxOffsetTest, CoreAdminCreateDiscoverTest, TestUninvertingReader, DistributedMLTComponentTest, CoreMergeIndexesAdminHandlerTest, BlobRepositoryCloudTest, TestPhraseSuggestions, TestFieldCacheSanityChecker, HdfsSyncSliceTest, TestJavabinTupleStreamParser, DocValuesTest, TestSolrDeletionPolicy1, HdfsBasicDistributedZkTest, TestIntervalFaceting, DocValuesNotIndexedTest, SchemaVersionSpecificBehaviorTest, HdfsNNFailoverTest, TestSubQueryTransformerDistrib, TestFilteredDocIdSet, DeleteReplicaTest, TestElisionMultitermQuery, DOMUtilTest, TestHashQParserPlugin, CopyFieldTest, SolrCoreTest, PrimitiveFieldTypeTest, UtilsToolTest, TestDistributedSearch, TestManagedSchemaThreadSafety, TimeZoneUtilsTest, ManagedSchemaRoundRobinCloudTest, TestJsonFacets, TestSubQueryTransformer, RuleEngineTest, TestPerFieldSimilarity, FileUtilsTest, DeleteStatusTest, TestFastLRUCache, TestCustomSort, TestUseDocValuesAsStored2, DistributedFacetPivotLongTailTest, TestSimpleQParserPlugin, TestNonDefinedSimilarityFactory, TestClassicSimilarityFactory, TestPKIAuthenticationPlugin, TestSQLHandler, TestCollectionAPI, SolrMetricManagerTest, RecoveryAfterSoftCommitTest, BlockJoinFacetRandomTest, TestFieldCacheSortRandom, TestCustomDocTransformer, SegmentsInfoRequestHandlerTest, TestReloadDeadlock, MultiTermTest, TestSort, TestBinaryField, DistributedQueueTest, CachingDirectoryFactoryTest, UUIDUpdateProcessorFallbackTest, TestImplicitCoreProperties, TestManagedResourceStorage, TestConfigSets, CdcrReplicationDistributedZkTest, HttpPartitionTest, HdfsDirectoryFactoryTest, TestCSVResponseWriter, BasicAuthIntegrationTest, DirectSolrSpellCheckerTest, DocValuesMissingTest, TestImpersonationWithHadoopAuth, DistributedDebugComponentTest, TestJettySolrRunner, TestTrackingShardHandlerFactory, HardAutoCommitTest, TestLFUCache, TestDocumentBuilder, ClusterStateUpdateTest, CloudExitableDirectoryReaderTest, DocumentBuilderTest, ReplicaListTransformerTest, TestFaceting, TestLeaderInitiatedRecoveryThread, TestStressReorder, SpatialRPTFieldTypeTest, TestSolrCLIRunExample, TestManagedStopFilterFactory, TestGeoJSONResponseWriter, TestSchemaNameResource, HdfsRecoveryZkTest, OverseerTest, TestComplexPhraseLeadingWildcard, DistributedSuggestComponentTest, BufferStoreTest, QueryElevationComponentTest, TestQuerySenderNoQuery, DistributedFacetExistsSmallTest, SimplePostToolTest, SpatialFilterTest, TestFieldCache, DistributedQueryComponentCustomSortTest, DirectoryFactoryTest, SOLR749Test, AtomicUpdatesTest, AnalysisAfterCoreReloadTest, MergeStrategyTest, TestOverriddenPrefixQueryForCustomFieldType, ChaosMonkeyNothingIsSafeTest, SolrXmlInZkTest, TestHashPartitioner, MBeansHandlerTest, GraphQueryTest, DocValuesMultiTest, TestBM25SimilarityFactory, RecoveryZkTest, ZkCLITest, TestRTGBase, SuggesterFSTTest, TestFunctionQuery, CurrencyFieldXmlFileTest, TestReloadAndDeleteDocs, ConnectionManagerTest, SortSpecParsingTest, TestConfigSetImmutable, TestSurroundQueryParser, TestSweetSpotSimilarityFactory, TestFieldCacheWithThreads, CloneFieldUpdateProcessorFactoryTest, TestBulkSchemaAPI, HighlighterTest, UniqFieldsUpdateProcessorFactoryTest, BJQParserTest, CollectionsAPIAsyncDistributedZkTest, SolrGraphiteReporterTest, ZkNodePropsTest, TestFreeTextSuggestions, HdfsTlogReplayBufferedWhileIndexingTest, TestClusterStateMutator, ShardRoutingTest, ZkControllerTest, TestSerializedLuceneMatchVersion, TestExceedMaxTermLength, DataDrivenBlockJoinTest, SimpleFacetsTest, TestInitParams, TestMacros, DisMaxRequestHandlerTest, BigEndianAscendingWordSerializerTest, OpenExchangeRatesOrgProviderTest, IndexSchemaRuntimeFieldTest, DateFieldTest, StatelessScriptUpdateProcessorFactoryTest, TestLRUCache, OutOfBoxZkACLAndCredentialsProvidersTest, SolrIndexMetricsTest]
   [junit4] Completed [317/676 (1!)] on J0 in 2.30s, 1 test, 1 failure <<< FAILURES!

[...truncated 104 lines...]
   [junit4] Suite: org.apache.solr.update.AutoCommitTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.AutoCommitTest_1FBDFA3E765036C4-001/init-core-data-001
   [junit4]   2> 1432874 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1432875 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 1432876 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1432919 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1432942 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 1433031 WARN  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1433033 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1433049 WARN  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1433051 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1433051 WARN  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1433054 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1433054 WARN  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1433054 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1433056 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1433085 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1433106 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1433193 WARN  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1433195 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1433201 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1433202 WARN  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1433202 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1433202 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.AutoCommitTest_1FBDFA3E765036C4-001/init-core-data-001/]
   [junit4]   2> 1433202 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1433203 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1433226 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1433226 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1433227 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1433227 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1433227 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=36.3212890625, floorSegmentMB=1.5751953125, forceMergeDeletesPctAllowed=1.1066196666772554, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1433228 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@c4407b4[collection1] main]
   [junit4]   2> 1433228 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 1433791 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1433807 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1433807 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 1433808 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 1433808 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1433808 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1433808 INFO  (coreLoadExecutor-5841-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1555927291591131136
   [junit4]   2> 1433808 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1433809 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 1433809 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 1433809 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 1433810 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 1433810 INFO  (searcherExecutor-5842-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@c4407b4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1433811 INFO  (SUITE-AutoCommitTest-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 1433813 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testMaxTime
   [junit4]   2> 1433814 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{deleteByQuery=*:* (-1555927291597422592)} 0 0
   [junit4]   2> 1433814 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1433851 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1433872 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1433960 WARN  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1433962 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1433969 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1433970 WARN  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1433978 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1433978 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.SolrMetricManager Replacing existing reporter 'default' in registry 'solr.core.collection1': [org.apache.solr.metrics.reporters.SolrJmxReporter@7797c787: domain = solr.core.collection1, service url = null, agent id = null]
   [junit4]   2> 1433982 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.AutoCommitTest_1FBDFA3E765036C4-001/init-core-data-001/]
   [junit4]   2> 1433982 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1434003 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1434003 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1434003 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@78c8f2e4[collection1] main]
   [junit4]   2> 1434003 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 1434004 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1434007 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1434007 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 1434008 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 1434008 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1434008 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1434008 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1434009 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 1434009 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 1434009 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 1434010 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 1434010 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@78c8f2e4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1434010 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1434012 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1434012 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@57b2c7ff[collection1] main]
   [junit4]   2> 1434012 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@138eb1e8
   [junit4]   2> 1434013 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@57b2c7ff[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1434017 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1434019 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1555927291812380672)]} 0 0
   [junit4]   2> 1434019 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1435519 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 1435527 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@78533c96[collection1] main]
   [junit4]   2> 1435528 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1435529 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@78533c96[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):c1)))}
   [junit4]   2> 1435776 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1555927293653680128)]} 0 1757
   [junit4]   2> 1435776 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 1435777 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:530&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1435777 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{delete=[529 (-1555927293655777280)]} 0 0
   [junit4]   2> 1435777 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 1437276 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 1437287 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@589328c7[collection1] main]
   [junit4]   2> 1437287 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1437288 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@589328c7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 1437532 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[550 (1555927295496028160)]} 0 3515
   [junit4]   2> 1437533 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1437533 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295497076736)]} 0 3516
   [junit4]   2> 1437534 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295497076737)]} 0 3516
   [junit4]   2> 1437534 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295498125312)]} 0 3516
   [junit4]   2> 1437534 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295498125313)]} 0 3517
   [junit4]   2> 1437534 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295498125314)]} 0 3517
   [junit4]   2> 1437534 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295498125315)]} 0 3517
   [junit4]   2> 1437535 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295499173888)]} 0 3517
   [junit4]   2> 1437535 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295499173889)]} 0 3517
   [junit4]   2> 1437535 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295499173890)]} 0 3517
   [junit4]   2> 1437535 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927295499173891)]} 0 3518
   [junit4]   2> 1437536 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:500&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1439033 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 1439041 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d50003e[collection1] main]
   [junit4]   2> 1439042 INFO  (commitScheduler-5850-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1439043 INFO  (searcherExecutor-5847-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6d50003e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1) Uninverting(_2(7.0.0):c11/9:delGen=1)))}
   [junit4]   2> 1439292 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[531 (1555927297340473344)]} 0 5275
   [junit4]   2> 1439292 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:500&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 1439293 INFO  (TEST-AutoCommitTest.testMaxTime-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxTime
   [junit4]   2> 1439296 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testMaxDocs
   [junit4]   2> 1439305 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@212005ce[collection1] realtime]
   [junit4]   2> 1439305 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{deleteByQuery=*:* (-1555927297345716224)} 0 9
   [junit4]   2> 1439306 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1439348 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1439379 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1439463 WARN  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1439465 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1439479 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1439480 WARN  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1439486 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1439487 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.AutoCommitTest_1FBDFA3E765036C4-001/init-core-data-001/]
   [junit4]   2> 1439487 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1439548 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1439548 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1439549 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@77d4e956[collection1] main]
   [junit4]   2> 1439549 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 1439550 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1439553 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1439557 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.1240234375, noCFSRatio=0.0]
   [junit4]   2> 1439558 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1439577 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 1439577 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 1439577 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1439577 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1439578 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1439578 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 1439579 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 1439579 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 1439579 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 1439580 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@77d4e956[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1439580 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@55784b77[collection1] main]
   [junit4]   2> 1439580 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@566962e
   [junit4]   2> 1439586 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1439591 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@55784b77[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1439591 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[0 (1555927297653997568)]} 0 1
   [junit4]   2> 1439592 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[1 (1555927297656094720)]} 0 1
   [junit4]   2> 1439592 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[2 (1555927297656094721)]} 0 2
   [junit4]   2> 1439592 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[3 (1555927297656094722)]} 0 2
   [junit4]   2> 1439592 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[4 (1555927297656094723)]} 0 2
   [junit4]   2> 1439592 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[5 (1555927297656094724)]} 0 2
   [junit4]   2> 1439593 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[6 (1555927297656094725)]} 0 2
   [junit4]   2> 1439593 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[7 (1555927297657143296)]} 0 2
   [junit4]   2> 1439593 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[8 (1555927297657143297)]} 0 2
   [junit4]   2> 1439593 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[9 (1555927297657143298)]} 0 3
   [junit4]   2> 1439593 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[10 (1555927297657143299)]} 0 3
   [junit4]   2> 1439593 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[11 (1555927297657143300)]} 0 3
   [junit4]   2> 1439593 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[12 (1555927297657143301)]} 0 3
   [junit4]   2> 1439594 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[13 (1555927297657143302)]} 0 3
   [junit4]   2> 1439594 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:1&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1439594 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[14 (1555927297658191872)]} 0 4
   [junit4]   2> 1439594 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[15 (1555927297658191873)]} 0 4
   [junit4]   2> 1439595 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:14&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1439595 ERROR (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=id:14&qt=standard&start=0&rows=20&version=2.2
   [junit4]   2> 1439595 ERROR (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.SolrTestCaseJ4 REQUEST FAILED: q=id:14&qt=standard&start=0&rows=20&version=2.2:java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]   2> 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> <response>
   [junit4]   2> <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]   2> </response>
   [junit4]   2> 
   [junit4]   2> 	request was:q=id:14&qt=standard&start=0&rows=20&version=2.2
   [junit4]   2> 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:811)
   [junit4]   2> 	at org.apache.solr.update.AutoCommitTest.testMaxDocs(AutoCommitTest.java:225)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1439595 INFO  (commitScheduler-5854-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 1439596 INFO  (TEST-AutoCommitTest.testMaxDocs-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testMaxDocs
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=AutoCommitTest -Dtests.method=testMaxDocs -Dtests.seed=1FBDFA3E765036C4 -Dtests.slow=true -Dtests.locale=cs-CZ -Dtests.timezone=Africa/Kigali -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.30s J0 | AutoCommitTest.testMaxDocs <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1FBDFA3E765036C4:A63C2CE15ABA324E]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:818)
   [junit4]    > 	at org.apache.solr.update.AutoCommitTest.testMaxDocs(AutoCommitTest.java:225)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound=1]
   [junit4]    > 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="0" start="0"></result>
   [junit4]    > </response>
   [junit4]    > 	request was:q=id:14&qt=standard&start=0&rows=20&version=2.2
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:811)
   [junit4]    > 	... 40 more
   [junit4]   2> 1439600 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testCommitWithin
   [junit4]   2> 1439606 INFO  (commitScheduler-5854-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7d9109a9[collection1] main]
   [junit4]   2> 1439606 INFO  (commitScheduler-5854-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1439607 INFO  (searcherExecutor-5851-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7d9109a9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(7.0.0):c16)))}
   [junit4]   2> 1439608 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5c8791ac[collection1] realtime]
   [junit4]   2> 1439608 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{deleteByQuery=*:* (-1555927297664483328)} 0 7
   [junit4]   2> 1439608 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 1439640 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1439660 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1439752 WARN  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 1439755 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 1439762 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1439763 WARN  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 1439770 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1439770 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.AutoCommitTest_1FBDFA3E765036C4-001/init-core-data-001/]
   [junit4]   2> 1439770 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@534f25f2
   [junit4]   2> 1439790 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1439790 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1439791 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6320fcd6[collection1] main]
   [junit4]   2> 1439791 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 1439792 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1439795 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1439795 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 1439795 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 1439795 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1439795 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1439796 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1439796 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 1439796 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 1439797 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 1439797 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 1439798 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6320fcd6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1439798 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=34.162109375, floorSegmentMB=1.5986328125, forceMergeDeletesPctAllowed=19.410479394063714, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6096923009002642
   [junit4]   2> 1439824 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 1439824 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@71800547[collection1] main]
   [junit4]   2> 1439824 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@355f038e
   [junit4]   2> 1439825 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@71800547[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1439831 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 1439846 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[529 (1555927297910898688)]} 0 10
   [junit4]   2> 1439846 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1443846 INFO  (commitScheduler-5858-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 1443854 INFO  (commitScheduler-5858-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@8a6deb8[collection1] main]
   [junit4]   2> 1443855 INFO  (commitScheduler-5858-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1443856 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@8a6deb8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_5(7.0.0):c1)))}
   [junit4]   2> 1443864 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[530 (1555927302134562816)]} 0 4030
   [junit4]   2> 1443864 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:530&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1443865 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {delete=[529 (-1555927302135611392)]} 0 4031
   [junit4]   2> 1443865 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 1445865 INFO  (commitScheduler-5858-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 1445873 INFO  (commitScheduler-5858-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@66aea8a7[collection1] main]
   [junit4]   2> 1445874 INFO  (commitScheduler-5858-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 1445874 INFO  (searcherExecutor-5855-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@66aea8a7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_6(7.0.0):c1)))}
   [junit4]   2> 1446125 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:529&qt=standard&start=0&rows=20&version=2.2} hits=0 status=0 QTime=0
   [junit4]   2> 1446125 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:530&qt=standard&start=0&rows=20&version=2.2} hits=1 status=0 QTime=0
   [junit4]   2> 1446126 INFO  (TEST-AutoCommitTest.testCommitWithin-seed#[1FBDFA3E765036C4]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] {add=[500 (1555927304506441728)]} 0 6293
   [junit4]   2> 1446126 INFO  (TEST-AutoCommit

[...truncated too long message...]

SMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	13	/solr/live_nodes
   [junit4]   2> 	13	/solr/collections
   [junit4]   2> 	3	/solr/overseer/queue
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 2939225 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[1FBDFA3E765036C4]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_1FBDFA3E765036C4-001
   [junit4]   2> Jan 08, 2017 4:23:21 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=lt, timezone=Asia/Vladivostok
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=118367960,total=530579456
   [junit4]   2> NOTE: All tests run in this JVM: [SystemInfoHandlerTest, TestDynamicFieldCollectionResource, HighlighterMaxOffsetTest, CoreAdminCreateDiscoverTest, TestUninvertingReader, DistributedMLTComponentTest, CoreMergeIndexesAdminHandlerTest, BlobRepositoryCloudTest, TestPhraseSuggestions, TestFieldCacheSanityChecker, HdfsSyncSliceTest, TestJavabinTupleStreamParser, DocValuesTest, TestSolrDeletionPolicy1, HdfsBasicDistributedZkTest, TestIntervalFaceting, DocValuesNotIndexedTest, SchemaVersionSpecificBehaviorTest, HdfsNNFailoverTest, TestSubQueryTransformerDistrib, TestFilteredDocIdSet, DeleteReplicaTest, TestElisionMultitermQuery, DOMUtilTest, TestHashQParserPlugin, CopyFieldTest, SolrCoreTest, PrimitiveFieldTypeTest, UtilsToolTest, TestDistributedSearch, TestManagedSchemaThreadSafety, TimeZoneUtilsTest, ManagedSchemaRoundRobinCloudTest, TestJsonFacets, TestSubQueryTransformer, RuleEngineTest, TestPerFieldSimilarity, FileUtilsTest, DeleteStatusTest, TestFastLRUCache, TestCustomSort, TestUseDocValuesAsStored2, DistributedFacetPivotLongTailTest, TestSimpleQParserPlugin, TestNonDefinedSimilarityFactory, TestClassicSimilarityFactory, TestPKIAuthenticationPlugin, TestSQLHandler, TestCollectionAPI, SolrMetricManagerTest, RecoveryAfterSoftCommitTest, BlockJoinFacetRandomTest, TestFieldCacheSortRandom, TestCustomDocTransformer, SegmentsInfoRequestHandlerTest, TestReloadDeadlock, MultiTermTest, TestSort, TestBinaryField, DistributedQueueTest, CachingDirectoryFactoryTest, UUIDUpdateProcessorFallbackTest, TestImplicitCoreProperties, TestManagedResourceStorage, TestConfigSets, CdcrReplicationDistributedZkTest, HttpPartitionTest, HdfsDirectoryFactoryTest, TestCSVResponseWriter, BasicAuthIntegrationTest, DirectSolrSpellCheckerTest, DocValuesMissingTest, TestImpersonationWithHadoopAuth, DistributedDebugComponentTest, TestJettySolrRunner, TestTrackingShardHandlerFactory, HardAutoCommitTest, TestLFUCache, TestDocumentBuilder, ClusterStateUpdateTest, CloudExitableDirectoryReaderTest, DocumentBuilderTest, ReplicaListTransformerTest, TestFaceting, TestLeaderInitiatedRecoveryThread, TestStressReorder, SpatialRPTFieldTypeTest, TestSolrCLIRunExample, TestManagedStopFilterFactory, TestGeoJSONResponseWriter, TestSchemaNameResource, HdfsRecoveryZkTest, OverseerTest, TestComplexPhraseLeadingWildcard, DistributedSuggestComponentTest, BufferStoreTest, QueryElevationComponentTest, TestQuerySenderNoQuery, DistributedFacetExistsSmallTest, SimplePostToolTest, SpatialFilterTest, TestFieldCache, DistributedQueryComponentCustomSortTest, DirectoryFactoryTest, SOLR749Test, AtomicUpdatesTest, AnalysisAfterCoreReloadTest, MergeStrategyTest, TestOverriddenPrefixQueryForCustomFieldType, ChaosMonkeyNothingIsSafeTest, SolrXmlInZkTest, TestHashPartitioner, MBeansHandlerTest, GraphQueryTest, DocValuesMultiTest, TestBM25SimilarityFactory, RecoveryZkTest, ZkCLITest, TestRTGBase, SuggesterFSTTest, TestFunctionQuery, CurrencyFieldXmlFileTest, TestReloadAndDeleteDocs, ConnectionManagerTest, SortSpecParsingTest, TestConfigSetImmutable, TestSurroundQueryParser, TestSweetSpotSimilarityFactory, TestFieldCacheWithThreads, CloneFieldUpdateProcessorFactoryTest, TestBulkSchemaAPI, HighlighterTest, UniqFieldsUpdateProcessorFactoryTest, BJQParserTest, CollectionsAPIAsyncDistributedZkTest, SolrGraphiteReporterTest, ZkNodePropsTest, TestFreeTextSuggestions, HdfsTlogReplayBufferedWhileIndexingTest, TestClusterStateMutator, ShardRoutingTest, ZkControllerTest, TestSerializedLuceneMatchVersion, TestExceedMaxTermLength, DataDrivenBlockJoinTest, SimpleFacetsTest, TestInitParams, TestMacros, DisMaxRequestHandlerTest, BigEndianAscendingWordSerializerTest, OpenExchangeRatesOrgProviderTest, IndexSchemaRuntimeFieldTest, DateFieldTest, StatelessScriptUpdateProcessorFactoryTest, TestLRUCache, OutOfBoxZkACLAndCredentialsProvidersTest, SolrIndexMetricsTest, MetricsHandlerTest, ClassificationUpdateProcessorIntegrationTest, TestLegacyNumericRangeQueryBuilder, DistributedSpellCheckComponentTest, CheckHdfsIndexTest, TestSolrConfigHandlerCloud, TestManagedSchema, TestCSVLoader, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, LeaderElectionIntegrationTest, LeaderElectionTest, ZkSolrClientTest, AutoCommitTest, TestRecovery, TestRealTimeGet, TestCoreContainer, TestSolr4Spatial, BadIndexSchemaTest, TestLazyCores, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SpellCheckCollatorTest, TestStressLucene, PolyFieldTest, SolrCoreCheckLockOnStartupTest, TestUpdate, TestAtomicUpdateErrorCases, TestRemoteStreaming, CacheHeaderTest, LukeRequestHandlerTest, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TestOmitPositions, TermsComponentTest, RequestHandlersTest, TermVectorComponentTest, MoreLikeThisHandlerTest, RequiredFieldsTest, LoggingHandlerTest, SolrPluginUtilsTest, TestCollationField, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, JsonLoaderTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, AlternateDirectoryTest, SolrIndexConfigTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, SampleTest, TestSearchPerf, NumericFieldsTest, MinimalSchemaTest, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest, TestDFRSimilarityFactory, TestIBSimilarityFactory, TestFastWriter, TestJmxMonitoredMap, ResourceLoaderTest, TestFastOutputStream, ScriptEngineTest, URLClassifyProcessorTest, PluginInfoTest, PrimUtilsTest, TestSolrJ, SliceStateTest, UUIDFieldTest, CircularListTest, CursorPagingTest, DistributedIntervalFacetingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestTolerantSearch, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, CollectionReloadTest, CollectionStateFormat2Test, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, CreateCollectionCleanupTest, CustomCollectionTest, DeleteShardTest, DistribJoinFromCollectionTest, ForceLeaderTest, MultiThreadedOCPTest, PeerSyncReplicationTest, SharedFSAutoReplicaFailoverTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCLIZkUtilsTest, SolrCloudExampleTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPivotFacet, TestCloudRecovery, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestDownShardTolerantSearch, TestExclusionRuleCollectionAccess, TestHdfsCloudBackupRestore, TestLockTree, TestMiniSolrCloudCluster, TestRandomRequestDistribution, TestReplicaProperties, TestRequestForwarding, TestRequestStatusCollectionAPI, TestSizeLimitedDistributedMap, TestStressLiveNodes, TlogReplayBufferedWhileIndexingTest, TriLevelCompositeIdRoutingTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, CoreSorterTest, ExitableDirectoryReaderTest, TestBackupRepositoryFactory, TestConfigSetProperties, TestCustomStream, TestSolrDynamicMBean, TestSolrXml, TestSolrCloudSnapshots, TestReqParamsAPI, TestSolrConfigHandlerConcurrent, DistributedQueryComponentOptimizationTest, SuggestComponentTest, TestDistributedStatsComponentCardinality, TestSlowCompositeReaderWrapper, SolrMetricReporterTest, SolrMetricsIntegrationTest, SolrJmxReporterTest, TestMacroExpander, SmileWriterTest, TestExportWriter, TestGraphMLResponseWriter, TestPushWriter, TestRawResponseWriter, TestSolrQueryResponse, TestChildDocTransformer, TestSubQueryTransformerCrossCore, TestRestManager, TestDefaultSearchFieldResource, TestFieldResource, TestSchemaSimilarityResource, TestSolrQueryParserResource, BadCopyFieldTest, BooleanFieldTest, ChangedSchemaMergeTest, EnumFieldTest, PreAnalyzedFieldManagedSchemaCloudTest, TestCloudSchemaless, TestReRankQParserPlugin, TestSmileRequest, TestSolr4Spatial2, TestSolrFieldCacheMBean, TestStandardQParsers, TestStressUserVersions, TestTrieFacet, TestXmlQParserPlugin, TestJsonFacetRefinement, TestOrdValues, TestScoreJoinQPScore, CloudMLTQParserTest, TestDFISimilarityFactory, TestDefaultStatsCache, TestDistribIDF, TestExactSharedStatsCache, TestExactStatsCache, BasicAuthStandaloneTest, PKIAuthenticationIntegrationTest, TestRuleBasedAuthorizationPlugin, TestSha256AuthenticationProvider, TestDelegationWithHadoopAuth]
   [junit4] Completed [648/676 (3!)] on J0 in 4.31s, 8 tests, 1 failure <<< FAILURES!

[...truncated 62534 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1063 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1063/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([F8B5996390C104A2:900AAC49405B164E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:280)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11074 lines...]
   [junit4] Suite: org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F8B5996390C104A2-001/init-core-data-001
   [junit4]   2> 477474 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[F8B5996390C104A2]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 477477 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[F8B5996390C104A2]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F8B5996390C104A2-001/tempDir-001
   [junit4]   2> 477477 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[F8B5996390C104A2]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 477477 INFO  (Thread-5537) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 477477 INFO  (Thread-5537) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 477577 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[F8B5996390C104A2]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:53889
   [junit4]   2> 477589 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15979fc1e0f0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 477592 INFO  (jetty-launcher-726-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 477592 INFO  (jetty-launcher-726-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 477595 INFO  (jetty-launcher-726-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@21d5d65a{/solr,null,AVAILABLE}
   [junit4]   2> 477598 INFO  (jetty-launcher-726-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3613497b{/solr,null,AVAILABLE}
   [junit4]   2> 477598 INFO  (jetty-launcher-726-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@20b5cd57{SSL,[ssl, http/1.1]}{127.0.0.1:41347}
   [junit4]   2> 477598 INFO  (jetty-launcher-726-thread-2) [    ] o.e.j.s.Server Started @480612ms
   [junit4]   2> 477598 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41347}
   [junit4]   2> 477598 INFO  (jetty-launcher-726-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3210da34{SSL,[ssl, http/1.1]}{127.0.0.1:62624}
   [junit4]   2> 477598 INFO  (jetty-launcher-726-thread-1) [    ] o.e.j.s.Server Started @480612ms
   [junit4]   2> 477598 ERROR (jetty-launcher-726-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62624}
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 477599 ERROR (jetty-launcher-726-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-07T17:31:23.144Z
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 477599 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-01-07T17:31:23.144Z
   [junit4]   2> 477613 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 477614 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 477642 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15979fc1e0f0002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 477644 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53889/solr
   [junit4]   2> 477645 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:53889/solr
   [junit4]   2> 477653 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15979fc1e0f0004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 477692 INFO  (jetty-launcher-726-thread-2) [n:127.0.0.1:41347_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:41347_solr
   [junit4]   2> 477693 INFO  (jetty-launcher-726-thread-1) [n:127.0.0.1:62624_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62624_solr
   [junit4]   2> 477694 INFO  (jetty-launcher-726-thread-2) [n:127.0.0.1:41347_solr    ] o.a.s.c.Overseer Overseer (id=97242990708195333-127.0.0.1:41347_solr-n_0000000000) starting
   [junit4]   2> 477696 INFO  (zkCallback-736-thread-1-processing-n:127.0.0.1:41347_solr) [n:127.0.0.1:41347_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 477699 INFO  (zkCallback-737-thread-1-processing-n:127.0.0.1:62624_solr) [n:127.0.0.1:62624_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 477709 INFO  (jetty-launcher-726-thread-2) [n:127.0.0.1:41347_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41347_solr
   [junit4]   2> 477711 INFO  (zkCallback-737-thread-1-processing-n:127.0.0.1:62624_solr) [n:127.0.0.1:62624_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 477718 INFO  (zkCallback-736-thread-1-processing-n:127.0.0.1:41347_solr) [n:127.0.0.1:41347_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 477770 INFO  (jetty-launcher-726-thread-2) [n:127.0.0.1:41347_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 477783 INFO  (jetty-launcher-726-thread-2) [n:127.0.0.1:41347_solr    ] o.a.s.s.HadoopAuthPlugin Params: {delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, type=simple, zk-dt-secret-manager.enable=true, signer.secret.provider=zookeeper}
   [junit4]   2> 477870 INFO  (jetty-launcher-726-thread-2) [n:127.0.0.1:41347_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 477896 INFO  (jetty-launcher-726-thread-1) [n:127.0.0.1:62624_solr    ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
   [junit4]   2> 477896 INFO  (jetty-launcher-726-thread-1) [n:127.0.0.1:62624_solr    ] o.a.s.s.HadoopAuthPlugin Params: {delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, type=simple, zk-dt-secret-manager.enable=true, signer.secret.provider=zookeeper}
   [junit4]   2> 477897 INFO  (jetty-launcher-726-thread-1) [n:127.0.0.1:62624_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 477914 INFO  (jetty-launcher-726-thread-1-EventThread) [n:127.0.0.1:62624_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 477914 INFO  (jetty-launcher-726-thread-2-EventThread) [n:127.0.0.1:41347_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 479041 INFO  (jetty-launcher-726-thread-1) [n:127.0.0.1:62624_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 479042 INFO  (jetty-launcher-726-thread-2) [n:127.0.0.1:41347_solr    ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
   [junit4]   2> 479043 INFO  (jetty-launcher-726-thread-1) [n:127.0.0.1:62624_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F8B5996390C104A2-001/tempDir-001/node1/.
   [junit4]   2> 479056 INFO  (jetty-launcher-726-thread-2) [n:127.0.0.1:41347_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F8B5996390C104A2-001/tempDir-001/node2/.
   [junit4]   2> 479525 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenSolrClient
   [junit4]   2> 479638 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 479661 INFO  (qtp591142464-8612) [n:127.0.0.1:62624_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CZAigFZngiqQAEDFLzxeQOVmrYGUqjXKgT--X1PiwQvB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 479661 INFO  (qtp591142464-8612) [n:127.0.0.1:62624_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CZAigFZngiqQAEDFLzxeQOVmrYGUqjXKgT--X1PiwQvB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 479662 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenSolrClient-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenSolrClient
   [junit4]   2> 479689 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenVerify
   [junit4]   2> 479710 INFO  (qtp591142464-8601) [n:127.0.0.1:62624_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CazigFZngiqswIDFOaA2kLsYhzszRHYI7HM6qdCaRlDB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 479710 INFO  (qtp591142464-8601) [n:127.0.0.1:62624_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CazigFZngiqswIDFOaA2kLsYhzszRHYI7HM6qdCaRlDB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 479716 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 479718 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 479727 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 479753 INFO  (qtp556375337-8610) [n:127.0.0.1:41347_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CazigFZngiqswIDFOaA2kLsYhzszRHYI7HM6qdCaRlDB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 479754 INFO  (qtp556375337-8610) [n:127.0.0.1:41347_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CazigFZngiqswIDFOaA2kLsYhzszRHYI7HM6qdCaRlDB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 479760 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 479762 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 479771 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 479773 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenVerify-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenVerify
   [junit4]   2> 479803 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancel
   [junit4]   2> 479827 ERROR (qtp591142464-8598) [    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_3]
   [junit4]   2> 479839 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 479840 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 479849 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 479855 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 479856 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 479863 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 479885 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 479886 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 479895 ERROR (qtp591142464-8612) [n:127.0.0.1:62624_solr    ] o.a.h.s.t.d.ZKDelegationTokenSecretManager No node in path [/ZKDTSMTokensRoot/DT_4]
   [junit4]   2> 479910 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 479920 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 479921 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancel-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancel
   [junit4]   2> 479964 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 480015 INFO  (qtp591142464-8601) [n:127.0.0.1:62624_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CfHigFZngirxwUDFCzNE4CdpOVMeJVlI9R2BBgkXSjYB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480015 INFO  (qtp591142464-8601) [n:127.0.0.1:62624_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CfHigFZngirxwUDFCzNE4CdpOVMeJVlI9R2BBgkXSjYB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480024 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480033 INFO  (qtp556375337-8609) [n:127.0.0.1:41347_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CfHigFZngirxwUDFCzNE4CdpOVMeJVlI9R2BBgkXSjYB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480033 INFO  (qtp556375337-8609) [n:127.0.0.1:41347_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CfHigFZngirxwUDFCzNE4CdpOVMeJVlI9R2BBgkXSjYB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480037 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 480038 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 480046 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480071 INFO  (qtp591142464-8612) [n:127.0.0.1:62624_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CgZigFZngisGQYDFNMm77Vm5icYU5Z3TM-MR84A75cLB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480071 INFO  (qtp591142464-8612) [n:127.0.0.1:62624_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CgZigFZngisGQYDFNMm77Vm5icYU5Z3TM-MR84A75cLB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480085 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480096 INFO  (qtp556375337-8608) [n:127.0.0.1:41347_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CgZigFZngisGQYDFNMm77Vm5icYU5Z3TM-MR84A75cLB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480096 INFO  (qtp556375337-8608) [n:127.0.0.1:41347_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CgZigFZngisGQYDFNMm77Vm5icYU5Z3TM-MR84A75cLB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480106 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480130 INFO  (qtp591142464-8601) [n:127.0.0.1:62624_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_ChUigFZngisVAcDFOME9uDYdx4ybep5BpajXzCVoQdAB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480130 INFO  (qtp591142464-8601) [n:127.0.0.1:62624_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_ChUigFZngisVAcDFOME9uDYdx4ybep5BpajXzCVoQdAB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480138 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480143 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 480144 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 480152 INFO  (qtp591142464-8603) [n:127.0.0.1:62624_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_ChUigFZngisVAcDFOME9uDYdx4ybep5BpajXzCVoQdAB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480152 INFO  (qtp591142464-8603) [n:127.0.0.1:62624_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_ChUigFZngisVAcDFOME9uDYdx4ybep5BpajXzCVoQdAB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480181 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480207 INFO  (qtp591142464-8602) [n:127.0.0.1:62624_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CigigFZngisoAgDFG17ka5BegLHhXtv9cPV-Qf93uY1B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480207 INFO  (qtp591142464-8602) [n:127.0.0.1:62624_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CigigFZngisoAgDFG17ka5BegLHhXtv9cPV-Qf93uY1B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480216 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480226 INFO  (qtp556375337-8605) [n:127.0.0.1:41347_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CigigFZngisoAgDFG17ka5BegLHhXtv9cPV-Qf93uY1B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480226 INFO  (qtp556375337-8605) [n:127.0.0.1:41347_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADZm9vA2ZvbwCKAVl5_CigigFZngisoAgDFG17ka5BegLHhXtv9cPV-Qf93uY1B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480233 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 480234 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 480244 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480246 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenew-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 480280 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancelFail-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenCancelFail
   [junit4]   2> 480294 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenCancelFail-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenCancelFail
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDelegationWithHadoopAuth -Dtests.method=testDelegationTokenCancelFail -Dtests.seed=F8B5996390C104A2 -Dtests.slow=true -Dtests.locale=he-IL -Dtests.timezone=Antarctica/Rothera -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.05s J0 | TestDelegationWithHadoopAuth.testDelegationTokenCancelFail <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<200> but was:<404>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F8B5996390C104A2:900AAC49405B164E]:0)
   [junit4]    > 	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.cancelDelegationToken(TestDelegationWithHadoopAuth.java:128)
   [junit4]    > 	at org.apache.solr.security.hadoop.TestDelegationWithHadoopAuth.testDelegationTokenCancelFail(TestDelegationWithHadoopAuth.java:280)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 480330 INFO  (TEST-TestDelegationWithHadoopAuth.testZNodePaths-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZNodePaths
   [junit4]   2> 480340 INFO  (TEST-TestDelegationWithHadoopAuth.testZNodePaths-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZNodePaths
   [junit4]   2> 480374 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationOtherOp
   [junit4]   2> 480379 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[F8B5996390C104A2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 480381 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[F8B5996390C104A2]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:53889/solr ready
   [junit4]   2> 480388 INFO  (qtp591142464-8612) [n:127.0.0.1:62624_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params op=someSolrOperation&action=LIST&user.name=foo&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 480388 INFO  (qtp591142464-8612) [n:127.0.0.1:62624_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={op=someSolrOperation&action=LIST&user.name=foo&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 480389 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=foo&p=foo&t=simple-dt&e=1483846285932&s=creLYF5dW3QhRx9WqMaAwyGdd4c="; Domain=127.0.0.1; Expires= , 08-   -2017 03:31:25 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 08- -2017 03:31:25 GMT
   [junit4]   2> 480391 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationOtherOp-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationOtherOp
   [junit4]   2> 480427 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 480436 WARN  (qtp591142464-8602) [n:127.0.0.1:62624_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar tries to renew a token with renewer foo
   [junit4]   2> 480436 WARN  (qtp591142464-8602) [n:127.0.0.1:62624_solr    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 480437 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480438 WARN  (qtp556375337-8606) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: bar tries to renew a token with renewer foo
   [junit4]   2> 480438 WARN  (qtp556375337-8606) [    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 480438 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480446 WARN  (qtp591142464-8601) [n:127.0.0.1:62624_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 480447 WARN  (qtp591142464-8601) [n:127.0.0.1:62624_solr    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 480448 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480449 WARN  (qtp556375337-8606) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: notRenewUser tries to renew a token with renewer renewUser
   [junit4]   2> 480449 WARN  (qtp556375337-8606) [    ] o.a.s.s.HadoopAuthPlugin Could not find org.apache.solr.security.authentication.requestcontinues
   [junit4]   2> 480449 WARN  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F8B5996390C104A2]) [    ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth=; Domain=127.0.0.1; Expires= , 01-   -1970 00:00:00 GMT; Secure; HttpOnly". Invalid 'expires' attribute: , 01- -1970 00:00:00 GMT
   [junit4]   2> 480449 INFO  (TEST-TestDelegationWithHadoopAuth.testDelegationTokenRenewFail-seed#[F8B5996390C104A2]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenewFail
   [junit4]   2> 480452 INFO  (jetty-launcher-726-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@20b5cd57{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 480453 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=380478430
   [junit4]   2> 480453 INFO  (jetty-launcher-726-thread-2) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@3210da34{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 480454 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1221494799
   [junit4]   2> 480460 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.c.Overseer Overseer (id=97242990708195333-127.0.0.1:41347_solr-n_0000000000) closing
   [junit4]   2> 480461 INFO  (OverseerStateUpdate-97242990708195333-127.0.0.1:41347_solr-n_0000000000) [n:127.0.0.1:41347_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:41347_solr
   [junit4]   2> 480464 INFO  (jetty-launcher-726-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 480465 ERROR (Thread[Thread-5555,5,TGRP-TestDelegationWithHadoopAuth]) [n:127.0.0.1:41347_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 480466 INFO  (zkCallback-737-thread-1-processing-n:127.0.0.1:62624_solr) [n:127.0.0.1:62624_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 480471 INFO  (jetty-launcher-726-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> 480472 ERROR (Thread[Thread-5554,5,TGRP-TestDelegationWithHadoopAuth]) [n:127.0.0.1:62624_solr    ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 480484 INFO  (jetty-launcher-726-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@21d5d65a{/solr,null,UNAVAILABLE}
   [junit4]   2> 480489 INFO  (jetty-launcher-726-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3613497b{/solr,null,UNAVAILABLE}
   [junit4]   2> 480492 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[F8B5996390C104A2]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53889 53889
   [junit4]   2> 480558 INFO  (Thread-5537) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53889 53889
   [junit4]   2> 480559 WARN  (Thread-5537) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/aliases.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/clusterprops.json
   [junit4]   2> 	11	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_7
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_6
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_9
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_8
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_1
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_3
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_2
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_5
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_4
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_12
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_11
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot/DT_10
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/live_nodes
   [junit4]   2> 	11	/solr/collections
   [junit4]   2> 	3	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/overseer/queue
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 	2	/solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
   [junit4]   2> 
   [junit4]   2> 480559 INFO  (SUITE-TestDelegationWithHadoopAuth-seed#[F8B5996390C104A2]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.security.hadoop.TestDelegationWithHadoopAuth_F8B5996390C104A2-001
   [junit4]   2> Jan 07, 2017 5:31:26 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1654, maxMBSortInHeap=6.916381429675816, sim=RandomSimilarity(queryNorm=false): {}, locale=he-IL, timezone=Antarctica/Rothera
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=97480160,total=233226240
   [junit4]   2> NOTE: All tests run in this JVM: [TestAnalyzeInfixSuggestions, TestSegmentSorting, CdcrRequestHandlerTest, ConcurrentDeleteAndCreateCollectionTest, TestBinaryField, DOMUtilTest, SolrMetricsIntegrationTest, TestSolrDeletionPolicy1, FastVectorHighlighterTest, TestExclusionRuleCollectionAccess, ActionThrottleTest, TestJmxIntegration, TestSlowCompositeReaderWrapper, OverseerRolesTest, TestCollectionAPI, TestManagedSchema, DeleteReplicaTest, CurrencyFieldXmlFileTest, SyncSliceTest, TestSortByMinMaxFunction, CopyFieldTest, CdcrUpdateLogTest, TestSerializedLuceneMatchVersion, CheckHdfsIndexTest, TestRecoveryHdfs, TestCopyFieldCollectionResource, TestSQLHandler, SolrGraphiteReporterTest, TestConfigSetsAPIExclusivity, TestRTGBase, TestManagedResourceStorage, TestCollationField, TestRandomDVFaceting, TestSubQueryTransformer, TestGraphTermsQParserPlugin, TestAddFieldRealTimeGet, DocExpirationUpdateProcessorFactoryTest, TestCloudSchemaless, TestFieldCacheReopen, PluginInfoTest, SmileWriterTest, TestCustomStream, HdfsNNFailoverTest, TestDelegationWithHadoopAuth]
   [junit4] Completed [128/675 (1!)] on J0 in 4.13s, 8 tests, 1 failure <<< FAILURES!

[...truncated 64185 lines...]


[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1062 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1062/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics

Error Message:
minorMerge: 3 expected:<4> but was:<3>

Stack Trace:
java.lang.AssertionError: minorMerge: 3 expected:<4> but was:<3>
	at __randomizedtesting.SeedInfo.seed([56E5B67BEE4E29C:C9BE66DB7E6A19A7]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics(SolrIndexMetricsTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:811)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:462)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12733 lines...]
   [junit4] Suite: org.apache.solr.update.SolrIndexMetricsTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SolrIndexMetricsTest_56E5B67BEE4E29C-001/init-core-data-001
   [junit4]   2> 2773222 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2773223 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 2773224 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2773247 WARN  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2773247 WARN  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 2773247 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2773257 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 2773347 WARN  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2773349 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2773360 WARN  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2773362 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@a27660a
   [junit4]   2> 2773362 WARN  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2773365 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@a27660a
   [junit4]   2> 2773365 WARN  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2773365 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@a27660a
   [junit4]   2> 2773366 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2773385 WARN  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2773386 WARN  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 2773386 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2773394 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2773488 WARN  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2773490 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2773497 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 2773498 WARN  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2773498 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@a27660a
   [junit4]   2> 2773498 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SolrIndexMetricsTest_56E5B67BEE4E29C-001/init-core-data-001/]
   [junit4]   2> 2773518 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2773518 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2773521 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@785c68ca[collection1] main]
   [junit4]   2> 2773521 WARN  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 2773521 INFO  (coreLoadExecutor-11579-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2773522 INFO  (searcherExecutor-11580-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@785c68ca[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2773523 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 2773528 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testIndexMetrics
   [junit4]   2> 2774331 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2774331 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@7907e48b
   [junit4]   2> 2774390 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.s.SolrIndexSearcher Opening [Searcher@5a1066f7[collection1] main]
   [junit4]   2> 2774391 INFO  (searcherExecutor-11580-thread-1) [    ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5a1066f7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(7.0.0):C300) Uninverting(_3(7.0.0):C100) Uninverting(_5(7.0.0):C100) Uninverting(_6(7.0.0):C100) Uninverting(_7(7.0.0):C100) Uninverting(_9(7.0.0):C100) Uninverting(_a(7.0.0):C100) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 2774391 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2774391 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib, /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 2774416 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 2774416 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergeFactor> is deprecated, configure it on the relevant <mergePolicyFactory> instead.
   [junit4]   2> 2774416 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2774425 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2774524 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 2774527 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 2774535 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1
   [junit4]   2> 2774536 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.m.r.SolrJmxReporter No serviceUrl or agentId was configured, using first MBeanServer.
   [junit4]   2> 2774551 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@a27660a
   [junit4]   2> 2774551 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.m.SolrMetricManager Replacing existing reporter 'default' in registry 'solr.core.collection1': [org.apache.solr.metrics.reporters.SolrJmxReporter@10e703ce: domain = solr.core.collection1, service url = null, agent id = null]
   [junit4]   2> 2774554 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SolrIndexMetricsTest_56E5B67BEE4E29C-001/init-core-data-001/]
   [junit4]   2> 2774565 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2774566 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2774566 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@32b4d453[collection1] main]
   [junit4]   2> 2774567 WARN  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 2774567 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2774568 INFO  (searcherExecutor-11584-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@32b4d453[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(7.0.0):C300) Uninverting(_3(7.0.0):C100) Uninverting(_5(7.0.0):C100) Uninverting(_6(7.0.0):C100) Uninverting(_7(7.0.0):C100) Uninverting(_9(7.0.0):C100) Uninverting(_a(7.0.0):C100) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 2774821 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 2774890 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@334fb035[collection1] main]
   [junit4]   2> 2774891 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@11bf3bc5
   [junit4]   2> 2774891 INFO  (searcherExecutor-11584-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@334fb035[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(7.0.0):C300) Uninverting(_8(7.0.0):C300) Uninverting(_c(7.0.0):C300) Uninverting(_b(7.0.0):C100)))}
   [junit4]   2> 2774891 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 2774895 INFO  (TEST-SolrIndexMetricsTest.testIndexMetrics-seed#[56E5B67BEE4E29C]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testIndexMetrics
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrIndexMetricsTest -Dtests.method=testIndexMetrics -Dtests.seed=56E5B67BEE4E29C -Dtests.slow=true -Dtests.locale=ar-YE -Dtests.timezone=America/Nome -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 1.37s J0 | SolrIndexMetricsTest.testIndexMetrics <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: minorMerge: 3 expected:<4> but was:<3>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([56E5B67BEE4E29C:C9BE66DB7E6A19A7]:0)
   [junit4]    > 	at org.apache.solr.update.SolrIndexMetricsTest.testIndexMetrics(SolrIndexMetricsTest.java:70)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2774897 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    x:collection1] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 2774897 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    x:collection1] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1232351088
   [junit4]   2> 2774898 INFO  (coreCloseExecutor-11588-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@aae0988
   [junit4]   2> 2774899 INFO  (coreCloseExecutor-11588-thread-1-processing-x:collection1) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.collection1
   [junit4]   2> 2774899 INFO  (SUITE-SolrIndexMetricsTest-seed#[56E5B67BEE4E29C]-worker) [    x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.node
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SolrIndexMetricsTest_56E5B67BEE4E29C-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {range_facet_l_dv=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), intDefault=PostingsFormat(name=LuceneVarGapFixedInterval), foo_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=Lucene50(blocksize=128), range_facet_i_dv=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_l=PostingsFormat(name=LuceneVarGapDocFreqInterval), timestamp=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene70), multiDefault=DocValuesFormat(name=Memory), intDefault=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Lucene70), foo_s=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Memory), range_facet_l=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=915, maxMBSortInHeap=7.881263486110305, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-YE, timezone=America/Nome
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_112 (64-bit)/cpus=3,threads=1,free=152589552,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrXml, TestCustomStream, TestFieldCacheVsDocValues, ChaosMonkeyNothingIsSafeTest, SchemaVersionSpecificBehaviorTest, StatelessScriptUpdateProcessorFactoryTest, DocumentAnalysisRequestHandlerTest, TestIBSimilarityFactory, TestPerFieldSimilarity, TestRandomDVFaceting, BigEndianAscendingWordDeserializerTest, HdfsBasicDistributedZkTest, TestXIncludeConfig, HdfsTlogReplayBufferedWhileIndexingTest, TestOnReconnectListenerSupport, SolrTestCaseJ4Test, TestPhraseSuggestions, HdfsRecoveryZkTest, TestCollationField, TestExtendedDismaxParser, FileUtilsTest, SegmentsInfoRequestHandlerTest, PluginInfoTest, RequiredFieldsTest, TestBlobHandler, CdcrReplicationDistributedZkTest, TestSort, PeerSyncTest, PingRequestHandlerTest, DistributedQueryComponentCustomSortTest, SampleTest, TestJettySolrRunner, MBeansHandlerTest, HdfsDirectoryFactoryTest, TestStressReorder, TestSolrJ, TestLockTree, SearchHandlerTest, HdfsChaosMonkeySafeLeaderTest, TestManagedStopFilterFactory, ClassificationUpdateProcessorTest, StandardRequestHandlerTest, TestExpandComponent, BadCopyFieldTest, CachingDirectoryFactoryTest, TestDefaultStatsCache, SynonymTokenizerTest, DocValuesTest, RestartWhileUpdatingTest, XmlUpdateRequestHandlerTest, TestConfigSets, TestSchemaNameResource, SolrCoreTest, TestDelegationWithHadoopAuth, CursorMarkTest, TestConfigSetImmutable, TestInitQParser, TestCSVResponseWriter, SimpleFacetsTest, TestSortByMinMaxFunction, PeerSyncWithIndexFingerprintCachingTest, TestExclusionRuleCollectionAccess, IndexSchemaRuntimeFieldTest, TestReqParamsAPI, TestUseDocValuesAsStored, FullHLLTest, LeaderElectionTest, TestAnalyzeInfixSuggestions, TlogReplayBufferedWhileIndexingTest, DeleteStatusTest, CollectionStateFormat2Test, SpatialRPTFieldTypeTest, CollectionsAPIAsyncDistributedZkTest, SolrCLIZkUtilsTest, DistributedMLTComponentTest, TestXmlQParser, TestGraphTermsQParserPlugin, TestFaceting, TestQueryWrapperFilter, TestConfigSetsAPIExclusivity, TestDocBasedVersionConstraints, ReplicaListTransformerTest, HdfsSyncSliceTest, SpatialFilterTest, SolrPluginUtilsTest, TestReplicaProperties, TermsComponentTest, TestClusterStateMutator, TestSolrDeletionPolicy1, DateFieldTest, TestRTimerTree, TestReplicationHandler, CopyFieldTest, HardAutoCommitTest, TestJmxIntegration, MetricUtilsTest, HdfsWriteToMultipleCollectionsTest, CursorPagingTest, TestFunctionQuery, PolyFieldTest, TestManagedSchema, ZkStateWriterTest, TestExactSharedStatsCache, RecoveryAfterSoftCommitTest, TestGeoJSONResponseWriter, HighlighterTest, TestJavabinTupleStreamParser, SpatialHeatmapFacetsTest, ConcurrentDeleteAndCreateCollectionTest, SSLMigrationTest, ClusterStateUpdateTest, TestFieldCacheReopen, DistanceUnitsTest, TestCustomDocTransformer, CdcrRequestHandlerTest, TestFieldTypeCollectionResource, DistributedQueueTest, QueryEqualityTest, TestSweetSpotSimilarityFactory, DocExpirationUpdateProcessorFactoryTest, CustomCollectionTest, PrimUtilsTest, BlockDirectoryTest, TestSolrFieldCacheMBean, TestManagedSchemaThreadSafety, TestSolrQueryParserResource, TestDynamicFieldCollectionResource, IndexBasedSpellCheckerTest, SimpleMLTQParserTest, TestCharFilters, DocValuesMultiTest, TestSolrCloudWithHadoopAuthPlugin, TestCursorMarkWithoutUniqueKey, BlockJoinFacetSimpleTest, SuggestComponentTest, TestWordDelimiterFilterFactory, RankQueryTest, BaseCdcrDistributedZkTest, AnalysisAfterCoreReloadTest, DeleteReplicaTest, TestFieldCacheSort, TestDistributedSearch, TestStressLucene, SolrGraphiteReporterTest, TestFieldSortValues, BlobRepositoryCloudTest, TestRequestStatusCollectionAPI, TestSlowCompositeReaderWrapper, TestCloudRecovery, SolrMetricManagerTest, SuggesterFSTTest, TestHdfsBackupRestoreCore, DistribDocExpirationUpdateProcessorTest, SolrRequestParserTest, TestSubQueryTransformerDistrib, TestHighlightDedupGrouping, TestComplexPhraseQParserPlugin, TestBinaryField, OverseerRolesTest, TestTolerantSearch, SimplePostToolTest, TestGroupingSearch, SolrSlf4jReporterTest, DirectUpdateHandlerTest, TestDistribDocBasedVersion, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, SyncSliceTest, ShardRoutingTest, TestRandomFaceting, TestRealTimeGet, TestRangeQuery, TestSolr4Spatial, BadIndexSchemaTest, TestLazyCores, SoftAutoCommitTest, TestFoldingMultitermQuery, SuggesterTest, SuggesterWFSTTest, TestPseudoReturnFields, FieldMutatingUpdateProcessorTest, SolrInfoMBeanTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, TestWriterPerf, PrimitiveFieldTypeTest, TestValueSourceCache, TermVectorComponentTest, TestIndexingPerformance, TestSolrQueryParser, ReturnFieldsTest, UniqFieldsUpdateProcessorFactoryTest, JsonLoaderTest, CSVRequestHandlerTest, SOLR749Test, TestQuerySenderListener, AlternateDirectoryTest, TestQuerySenderNoQuery, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, MultiTermTest, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestLMJelinekMercerSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, TestJmxMonitoredMap, TimeZoneUtilsTest, TestFastOutputStream, ScriptEngineTest, OpenExchangeRatesOrgProviderTest, TestFastLRUCache, PreAnalyzedFieldTest, TestSystemIdResolver, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, TestRTGBase, ActionThrottleTest, AliasIntegrationTest, CleanupOldIndexTest, DeleteInactiveReplicaTest, DistribCursorPagingTest, HttpPartitionTest, LeaderInitiatedRecoveryOnCommitTest, MigrateRouteKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerStatusTest, OverseerTaskQueueTest, ReplicationFactorTest, RollingRestartTest, SharedFSAutoReplicaFailoverUtilsTest, SimpleCollectionCreateDeleteTest, SolrCloudExampleTest, SolrXmlInZkTest, TestCloudDeleteByQuery, TestCloudPivotFacet, TestCollectionAPI, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestDownShardTolerantSearch, TestLeaderElectionWithEmptyReplica, TestLeaderInitiatedRecoveryThread, TestMiniSolrCloudCluster, TestMiniSolrCloudClusterSSL, TestRebalanceLeaders, TestSizeLimitedDistributedMap, TestTolerantUpdateProcessorCloud, VMParamsZkACLAndCredentialsProvidersTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsNNFailoverTest, HdfsThreadLeakTest, ZkStateReaderTest, RuleEngineTest, CoreSorterTest, DirectoryFactoryTest, TestBackupRepositoryFactory, TestConfigOverlay, TestConfigSetProperties, TestImplicitCoreProperties, TestInitParams, TestNRTOpen, TestReloadAndDeleteDocs, TestShardHandlerFactory, TestSolrConfigHandler, TestSolrDynamicMBean, TestSolrCoreSnapshots, RequestLoggingTest, TestRestoreCore, TestSQLHandler, CoreAdminCreateDiscoverTest, CoreAdminRequestStatusTest, CoreMergeIndexesAdminHandlerTest, InfoHandlerTest, MetricsHandlerTest, SecurityConfHandlerTest, StatsReloadRaceTest, DistributedDebugComponentTest, DistributedFacetExistsSmallTest, DistributedFacetPivotLargeTest, DistributedFacetPivotLongTailTest, DistributedFacetPivotSmallTest, DistributedSuggestComponentTest, JavabinLoaderTest, HighlighterMaxOffsetTest, TestUnifiedSolrHighlighter, WrapperMergePolicyFactoryTest, CheckHdfsIndexTest, TestCollationFieldDocValues, TestManagedSchemaAPI, TestUseDocValuesAsStored2, AnalyticsMergeStrategyTest, MergeStrategyTest, SortSpecParsingTest, TestAddFieldRealTimeGet, TestCollapseQParserPlugin, TestFilteredDocIdSet, TestHashQParserPlugin, TestLegacyNumericRangeQueryBuilder, TestMaxScoreQueryParser, TestNoOpRegenerator, TestRecoveryHdfs, TestMinMaxOnMultiValuedField, BJQParserTest, BlockJoinFacetDistribTest, BlockJoinFacetRandomTest, TestScoreJoinQPNoScore, TestJsonRequest, TestDFISimilarityFactory, TestPerFieldSimilarityWithDefaultOverride, TestExactStatsCache, TestLRUStatsCache, TestAuthorizationFramework, HdfsLockFactoryTest, TestDocTermOrds, TestFieldCacheSanityChecker, TestFieldCacheSortRandom, TestFieldCacheWithThreads, TestLegacyFieldCache, TestNumericTerms64, TestUninvertingReader, AddBlockUpdateTest, AnalysisErrorHandlingTest, CdcrUpdateLogTest, SolrIndexMetricsTest]
   [junit4] Completed [658/675 (1!)] on J0 in 1.69s, 1 test, 1 failure <<< FAILURES!

[...truncated 62498 lines...]