You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2016/07/05 06:49:43 UTC

[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 244 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/244/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.overseer.ZkStateWriterTest.testExternalModificationToStateFormat2

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([4C9C2773E19413A3:3D4133A7803FEF23]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.overseer.ZkStateWriterTest.testExternalModificationToStateFormat2(ZkStateWriterTest.java:322)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10874 lines...]
   [junit4] Suite: org.apache.solr.cloud.overseer.ZkStateWriterTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.overseer.ZkStateWriterTest_4C9C2773E19413A3-001/init-core-data-001
   [junit4]   2> 599377 INFO  (SUITE-ZkStateWriterTest-seed#[4C9C2773E19413A3]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 599378 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testExternalModificationToSharedClusterState
   [junit4]   2> 599378 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 599379 INFO  (Thread-1730) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 599379 INFO  (Thread-1730) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 599479 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer start zk server on port:36367
   [junit4]   2> 599479 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599479 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599482 INFO  (zkCallback-1262-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@348b1c0e name:ZooKeeperConnection Watcher:127.0.0.1:36367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599482 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599482 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599484 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599485 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599486 INFO  (zkCallback-1263-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@23caf70a name:ZooKeeperConnection Watcher:127.0.0.1:36367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599487 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599487 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599487 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 599492 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599492 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599497 INFO  (zkCallback-1264-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@64d01e2c name:ZooKeeperConnection Watcher:127.0.0.1:36367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599497 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599498 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599498 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 599500 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 599502 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 599503 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 599506 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 599507 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 599508 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 599509 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 599510 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 599512 INFO  (zkCallback-1264-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 599512 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c2
   [junit4]   2> 599514 INFO  (zkCallback-1264-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 599515 INFO  (zkCallback-1264-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 599519 INFO  (zkCallback-1264-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 599522 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36367 36367
   [junit4]   2> 599542 INFO  (Thread-1730) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36367 36367
   [junit4]   2> 599544 INFO  (TEST-ZkStateWriterTest.testExternalModificationToSharedClusterState-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testExternalModificationToSharedClusterState
   [junit4]   2> 599547 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSingleExternalCollection
   [junit4]   2> 599547 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 599547 INFO  (Thread-1731) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 599547 INFO  (Thread-1731) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 599647 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer start zk server on port:63169
   [junit4]   2> 599647 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599648 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599653 INFO  (zkCallback-1266-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2ef54fc2 name:ZooKeeperConnection Watcher:127.0.0.1:63169 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599653 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599653 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599662 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599663 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599673 INFO  (zkCallback-1267-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2dd2ff3d name:ZooKeeperConnection Watcher:127.0.0.1:63169 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599673 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599674 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599674 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 599681 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599682 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599685 INFO  (zkCallback-1268-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6a1e303f name:ZooKeeperConnection Watcher:127.0.0.1:63169/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599685 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599685 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599687 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 599688 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 599690 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 599691 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 599692 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 599693 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 599694 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 599695 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 599696 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 599698 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 599699 INFO  (zkCallback-1268-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 599703 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63169 63169
   [junit4]   2> 599863 INFO  (Thread-1731) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63169 63169
   [junit4]   2> 599866 INFO  (TEST-ZkStateWriterTest.testSingleExternalCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSingleExternalCollection
   [junit4]   2> 599867 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testExternalModificationToStateFormat2
   [junit4]   2> 599868 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 599868 INFO  (Thread-1732) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 599868 INFO  (Thread-1732) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 599968 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer start zk server on port:58344
   [junit4]   2> 599969 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599969 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599972 INFO  (zkCallback-1270-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3ae4c432 name:ZooKeeperConnection Watcher:127.0.0.1:58344 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599973 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599973 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599976 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599976 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599978 INFO  (zkCallback-1271-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@642fc965 name:ZooKeeperConnection Watcher:127.0.0.1:58344 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599979 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599979 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599979 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 599982 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 599983 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 599986 INFO  (zkCallback-1272-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@72b74822 name:ZooKeeperConnection Watcher:127.0.0.1:58344/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599986 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 599986 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 599987 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 599988 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 599989 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 599990 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 599992 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 599993 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 599995 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 599995 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 599998 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 600000 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c2
   [junit4]   2> 600001 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c2/state.json
   [junit4]   2> 600004 ERROR (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]-EventThread) [    ] o.a.z.ClientCnxn Error while calling watcher 
   [junit4]   2> java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$8/1041950479@6cb70980 rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@688890ce[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 1]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:214)
   [junit4]   2> 	at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:267)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 600005 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:58344 58344
   [junit4]   2> 600038 INFO  (Thread-1732) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:58344 58344
   [junit4]   2> 600039 INFO  (TEST-ZkStateWriterTest.testExternalModificationToStateFormat2-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testExternalModificationToStateFormat2
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ZkStateWriterTest -Dtests.method=testExternalModificationToStateFormat2 -Dtests.seed=4C9C2773E19413A3 -Dtests.slow=true -Dtests.locale=es-ES -Dtests.timezone=Etc/GMT-13 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.18s J1 | ZkStateWriterTest.testExternalModificationToStateFormat2 <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4C9C2773E19413A3:3D4133A7803FEF23]:0)
   [junit4]    > 	at org.apache.solr.cloud.overseer.ZkStateWriterTest.testExternalModificationToStateFormat2(ZkStateWriterTest.java:322)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 600046 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSingleLegacyCollection
   [junit4]   2> 600046 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 600046 INFO  (Thread-1733) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 600046 INFO  (Thread-1733) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 600146 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer start zk server on port:60252
   [junit4]   2> 600146 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 600147 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 600151 INFO  (zkCallback-1274-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@22e7503d name:ZooKeeperConnection Watcher:127.0.0.1:60252 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 600151 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 600151 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 600155 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 600155 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 600164 INFO  (zkCallback-1275-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@481b2ec2 name:ZooKeeperConnection Watcher:127.0.0.1:60252 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 600164 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 600164 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 600164 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 600175 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 600185 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 600187 INFO  (zkCallback-1276-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3eaa28fd name:ZooKeeperConnection Watcher:127.0.0.1:60252/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 600187 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 600188 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 600189 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 600191 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 600193 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 600195 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 600196 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 600197 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 600198 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 600199 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 600201 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 600203 INFO  (zkCallback-1276-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 600204 INFO  (zkCallback-1276-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 600207 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60252 60252
   [junit4]   2> 600232 INFO  (Thread-1733) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60252 60252
   [junit4]   2> 600234 INFO  (TEST-ZkStateWriterTest.testSingleLegacyCollection-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSingleLegacyCollection
   [junit4]   2> 600237 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testZkStateWriterBatching
   [junit4]   2> 600238 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 600238 INFO  (Thread-1734) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 600238 INFO  (Thread-1734) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 600338 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer start zk server on port:63386
   [junit4]   2> 600338 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 600339 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 600341 INFO  (zkCallback-1278-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@436ec675 name:ZooKeeperConnection Watcher:127.0.0.1:63386 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 600341 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 600341 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 600343 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 600343 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 600346 INFO  (zkCallback-1279-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@27dd042a name:ZooKeeperConnection Watcher:127.0.0.1:63386 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 600346 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 600346 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 600346 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 600350 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 600350 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 600352 INFO  (zkCallback-1280-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2ad21978 name:ZooKeeperConnection Watcher:127.0.0.1:63386/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 600352 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 600352 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 600353 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 600355 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 600357 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 600359 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 600361 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 600362 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 600363 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 600364 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 600366 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 600367 INFO  (zkCallback-1280-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 600368 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c2
   [junit4]   2> 600369 INFO  (zkCallback-1280-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 600370 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 600372 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63386 63386
   [junit4]   2> 600387 INFO  (Thread-1734) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63386 63386
   [junit4]   2> 600389 INFO  (TEST-ZkStateWriterTest.testZkStateWriterBatching-seed#[4C9C2773E19413A3]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testZkStateWriterBatching
   [junit4]   2> 600390 INFO  (SUITE-ZkStateWriterTest-seed#[4C9C2773E19413A3]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.overseer.ZkStateWriterTest_4C9C2773E19413A3-001
   [junit4]   2> Jul 05, 2016 5:59:35 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=566, maxMBSortInHeap=7.257207235386618, sim=ClassicSimilarity, locale=es-ES, timezone=Etc/GMT-13
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=3,threads=1,free=143924520,total=325775360
   [junit4]   2> NOTE: All tests run in this JVM: [CloneFieldUpdateProcessorFactoryTest, CopyFieldTest, RuleEngineTest, FileUtilsTest, SolrIndexSplitterTest, ShardRoutingCustomTest, TestPKIAuthenticationPlugin, TestFoldingMultitermQuery, SolrCmdDistributorTest, BlockJoinFacetRandomTest, AtomicUpdatesTest, DistributedFacetPivotWhiteBoxTest, TestDynamicFieldCollectionResource, CdcrReplicationHandlerTest, CollectionReloadTest, DistributedTermsComponentTest, DefaultValueUpdateProcessorTest, TestRebalanceLeaders, DateMathParserTest, TestUninvertingReader, TestFieldSortValues, BJQParserTest, OpenCloseCoreStressTest, TestReplicationHandlerBackup, MultiTermTest, LeaderInitiatedRecoveryOnShardRestartTest, TestStressRecovery, TestRealTimeGet, SolrCloudExampleTest, TestRTGBase, TestBadConfig, AsyncCallRequestStatusResponseTest, AutoCommitTest, TestSolr4Spatial2, DeleteReplicaTest, JavabinLoaderTest, TestSolrConfigHandlerCloud, ConfigSetsAPITest, TestGraphTermsQParserPlugin, OverseerModifyCollectionTest, TestLMJelinekMercerSimilarityFactory, ZkSolrClientTest, TestValueSourceCache, SmileWriterTest, DistributedFacetPivotLargeTest, ChaosMonkeySafeLeaderTest, TestScoreJoinQPScore, SharedFSAutoReplicaFailoverTest, CleanupOldIndexTest, TestCloudManagedSchema, QueryResultKeyTest, TestDefaultStatsCache, SuggestComponentContextFilterQueryTest, TestBulkSchemaAPI, TestSizeLimitedDistributedMap, ZkStateWriterTest]
   [junit4] Completed [107/619 (1!)] on J1 in 2.04s, 5 tests, 1 failure <<< FAILURES!

[...truncated 1666 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:740: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:684: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:59: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build.xml:233: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/common-build.xml:530: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:1427: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:984: There were test failures: 619 suites (10 ignored), 2595 tests, 1 failure, 86 ignored (71 assumptions) [seed: 4C9C2773E19413A3]

Total time: 81 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 249 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/249/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.cloud.overseer.ZkStateReaderTest.testExternalCollectionWatchedNotWatched

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([11D44301646A6434:1A6FB22D3533929D]:0)
	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testExternalCollectionWatchedNotWatched(ZkStateReaderTest.java:167)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10684 lines...]
   [junit4] Suite: org.apache.solr.cloud.overseer.ZkStateReaderTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.overseer.ZkStateReaderTest_11D44301646A6434-001/init-core-data-001
   [junit4]   2> 124425 INFO  (SUITE-ZkStateReaderTest-seed#[11D44301646A6434]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 124441 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testExternalCollectionWatchedNotWatched
   [junit4]   2> 124441 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 124441 INFO  (Thread-328) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 124441 INFO  (Thread-328) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 124541 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer start zk server on port:35032
   [junit4]   2> 124541 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 124542 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 124545 INFO  (zkCallback-112-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1ca114b9 name:ZooKeeperConnection Watcher:127.0.0.1:35032 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 124545 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 124548 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 124553 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 124553 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 124567 INFO  (zkCallback-113-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@66684da8 name:ZooKeeperConnection Watcher:127.0.0.1:35032 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 124567 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 124567 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 124567 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 124578 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 124579 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 124580 INFO  (zkCallback-114-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@42118caf name:ZooKeeperConnection Watcher:127.0.0.1:35032/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 124580 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 124581 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 124581 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 124582 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 124584 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 124586 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 124587 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 124588 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 124589 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 124589 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 124591 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 124592 INFO  (zkCallback-114-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 124593 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 124596 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35032 35032
   [junit4]   2> 124618 INFO  (Thread-328) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35032 35032
   [junit4]   2> 124664 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testExternalCollectionWatchedNotWatched
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ZkStateReaderTest -Dtests.method=testExternalCollectionWatchedNotWatched -Dtests.seed=11D44301646A6434 -Dtests.slow=true -Dtests.locale=ar -Dtests.timezone=America/Resolute -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.24s J0 | ZkStateReaderTest.testExternalCollectionWatchedNotWatched <<<
   [junit4]    > Throwable #1: java.lang.NullPointerException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([11D44301646A6434:1A6FB22D3533929D]:0)
   [junit4]    > 	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testExternalCollectionWatchedNotWatched(ZkStateReaderTest.java:167)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 124675 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithExplicitRefresh
   [junit4]   2> 124675 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 124676 INFO  (Thread-329) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 124676 INFO  (Thread-329) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 124776 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer start zk server on port:35969
   [junit4]   2> 124777 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 124777 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 124791 INFO  (zkCallback-116-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@134f1310 name:ZooKeeperConnection Watcher:127.0.0.1:35969 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 124792 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 124792 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 124793 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 124794 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 124797 INFO  (zkCallback-117-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1d61d188 name:ZooKeeperConnection Watcher:127.0.0.1:35969 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 124797 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 124797 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 124797 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 124807 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 124807 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 124809 INFO  (zkCallback-118-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@baea514 name:ZooKeeperConnection Watcher:127.0.0.1:35969/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 124809 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 124809 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 124810 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 124811 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 124813 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 124814 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 124816 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 124817 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 124817 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 124818 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 124821 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 124821 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 124824 INFO  (zkCallback-118-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 124826 INFO  (zkCallback-118-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 124829 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 124829 INFO  (zkCallback-118-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 124830 INFO  (zkCallback-118-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 124836 INFO  (zkCallback-118-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 124836 WARN  (zkCallback-118-thread-2) [    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /clusterstate.json]
   [junit4]   2> 124842 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35969 35969
   [junit4]   2> 125126 INFO  (Thread-329) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35969 35969
   [junit4]   2> 125133 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithExplicitRefresh
   [junit4]   2> 125141 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithTimeDelayLazy
   [junit4]   2> 125141 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 125142 INFO  (Thread-330) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 125142 INFO  (Thread-330) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 125242 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer start zk server on port:46142
   [junit4]   2> 125242 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125242 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125284 INFO  (zkCallback-120-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@76562c24 name:ZooKeeperConnection Watcher:127.0.0.1:46142 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125284 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125284 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125286 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125287 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125288 INFO  (zkCallback-121-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4d7555ba name:ZooKeeperConnection Watcher:127.0.0.1:46142 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125288 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125288 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125288 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 125292 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125293 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125295 INFO  (zkCallback-122-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4f3e194e name:ZooKeeperConnection Watcher:127.0.0.1:46142/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125295 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125295 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125296 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 125298 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 125299 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 125305 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 125307 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 125308 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 125309 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 125310 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 125312 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 125316 INFO  (zkCallback-122-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 125320 INFO  (zkCallback-122-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 125327 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 125329 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 125330 INFO  (zkCallback-122-thread-2) [    ] o.a.s.c.c.ZkStateReader Uninteresting collection c1
   [junit4]   2> 125330 INFO  (zkCallback-122-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 125332 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 125336 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46142 46142
   [junit4]   2> 125362 INFO  (Thread-330) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46142 46142
   [junit4]   2> 125366 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithTimeDelayLazy
   [junit4]   2> 125380 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testWatchedCollectionCreation
   [junit4]   2> 125380 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 125380 INFO  (Thread-331) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 125380 INFO  (Thread-331) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 125480 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer start zk server on port:53190
   [junit4]   2> 125480 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125481 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125489 INFO  (zkCallback-124-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@62eeac16 name:ZooKeeperConnection Watcher:127.0.0.1:53190 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125490 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125490 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125493 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125493 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125495 INFO  (zkCallback-125-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2d9d00f9 name:ZooKeeperConnection Watcher:127.0.0.1:53190 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125495 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125495 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125495 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 125502 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125502 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125503 INFO  (zkCallback-126-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@51dc8748 name:ZooKeeperConnection Watcher:127.0.0.1:53190/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125504 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125504 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125504 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 125506 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 125508 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 125510 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 125511 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 125513 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 125513 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 125514 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 125516 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 125516 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 125518 INFO  (zkCallback-126-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 125520 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 125521 INFO  (zkCallback-126-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 125523 INFO  (zkCallback-126-thread-2) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 125524 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x155c3c3e4470002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 125526 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53190 53190
   [junit4]   2> 125578 INFO  (Thread-331) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53190 53190
   [junit4]   2> 125585 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testWatchedCollectionCreation
   [junit4]   2> 125608 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithTimeDelay
   [junit4]   2> 125608 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 125609 INFO  (Thread-332) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 125609 INFO  (Thread-332) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 125709 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer start zk server on port:45670
   [junit4]   2> 125709 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125709 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125714 INFO  (zkCallback-128-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3a6ff55 name:ZooKeeperConnection Watcher:127.0.0.1:45670 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125715 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125715 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125717 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125717 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125718 INFO  (zkCallback-129-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7f5b8aaf name:ZooKeeperConnection Watcher:127.0.0.1:45670 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125718 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125719 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125719 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 125722 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125722 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125725 INFO  (zkCallback-130-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6595847e name:ZooKeeperConnection Watcher:127.0.0.1:45670/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125725 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125725 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125726 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 125727 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 125729 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 125730 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 125732 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 125733 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 125734 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 125735 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 125737 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 125737 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 125738 INFO  (zkCallback-130-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 125740 INFO  (zkCallback-130-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 125741 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 125741 INFO  (zkCallback-130-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 125742 INFO  (zkCallback-130-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 125742 INFO  (zkCallback-130-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 125746 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45670 45670
   [junit4]   2> 125753 INFO  (Thread-332) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45670 45670
   [junit4]   2> 125754 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithTimeDelay
   [junit4]   2> 125757 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithExplicitRefreshLazy
   [junit4]   2> 125758 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 125758 INFO  (Thread-333) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 125758 INFO  (Thread-333) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 125858 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer start zk server on port:63588
   [junit4]   2> 125858 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125859 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125861 INFO  (zkCallback-132-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@73cb3c88 name:ZooKeeperConnection Watcher:127.0.0.1:63588 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125861 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125861 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125868 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125868 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125872 INFO  (zkCallback-133-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6f7eeed6 name:ZooKeeperConnection Watcher:127.0.0.1:63588 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125872 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125872 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125872 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 125876 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 125876 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 125877 INFO  (zkCallback-134-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@46885826 name:ZooKeeperConnection Watcher:127.0.0.1:63588/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 125877 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 125878 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 125878 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 125881 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 125883 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 125884 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 125886 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 125887 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 125888 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 125889 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 125890 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 125892 INFO  (zkCallback-134-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 125895 INFO  (zkCallback-134-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 125897 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 125898 INFO  (zkCallback-134-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 125902 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63588 63588
   [junit4]   2> 125930 INFO  (Thread-333) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63588 63588
   [junit4]   2> 125964 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[11D44301646A6434]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithExplicitRefreshLazy
   [junit4]   2> 125965 INFO  (SUITE-ZkStateReaderTest-seed#[11D44301646A6434]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.overseer.ZkStateReaderTest_11D44301646A6434-001
   [junit4]   2> Jul 07, 2016 5:10:43 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=1955, maxMBSortInHeap=7.417673077891706, sim=RandomSimilarity(queryNorm=true,coord=yes): {}, locale=ar, timezone=America/Resolute
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=3,threads=1,free=18577224,total=127488000
   [junit4]   2> NOTE: All tests run in this JVM: [StressHdfsTest, ZkControllerTest, SolrCLIZkUtilsTest, HighlighterTest, UUIDFieldTest, TermsComponentTest, CdcrUpdateLogTest, TestFieldCacheReopen, ExternalFileFieldSortTest, TestJmxIntegration, TestRemoteStreaming, SystemInfoHandlerTest, HdfsRestartWhileUpdatingTest, HLLUtilTest, ZkStateReaderTest]
   [junit4] Completed [41/621 (1!)] on J0 in 2.09s, 6 tests, 1 error <<< FAILURES!

[...truncated 1883 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:740: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:684: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:59: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build.xml:233: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/common-build.xml:530: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:1427: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:984: There were test failures: 621 suites (10 ignored), 2642 tests, 1 error, 103 ignored (88 assumptions) [seed: 11D44301646A6434]

Total time: 77 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



Re: [JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 248 - Still Failing!

Posted by David Smiley <ds...@apache.org>.
Woops; good catch!  I was confused when I deconflicted the cherry-pick.
It's nice our smoke tester catches this stuff :-)  I'll go fix now.

On Wed, Jul 6, 2016 at 4:24 PM Steve Rowe <sa...@gmail.com> wrote:

> Hi David,
>
> Looks like you meant to move the Bug Fixes section in Lucene’s CHANGES.txt
> down below the New Features section, but you left the original at the top
> when you created the new one?
>
> --
> Steve
> www.lucidworks.com
>
> > On Jul 6, 2016, at 4:17 PM, Policeman Jenkins Server <
> jenkins@thetaphi.de> wrote:
> >
> > Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/248/
> > Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
> >
> > All tests passed
> >
> > Build Log:
> > [...truncated 53916 lines...]
> > changes-to-html:
> >    [mkdir] Created dir:
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/docs/changes
> >      [get] Getting:
> https://issues.apache.org/jira/rest/api/2/project/LUCENE
> >      [get] To:
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/docs/changes/jiraVersionList.json
> >     [exec] Section 'Bug Fixes' appears more than once under release
> '6.2.0' at
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/site/changes/
> changes2html.pl line 135.
> >
> > BUILD FAILED
> > /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:740:
> The following error occurred while executing this line:
> > /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:101:
> The following error occurred while executing this line:
> >
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build.xml:138:
> The following error occurred while executing this line:
> >
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build.xml:479:
> The following error occurred while executing this line:
> >
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:2498:
> exec returned: 255
> >
> > Total time: 95 minutes 20 seconds
> > Build step 'Invoke Ant' marked build as failure
> > Archiving artifacts
> > [WARNINGS] Skipping publisher since build result is FAILURE
> > Recording test results
> > Email was triggered for: Failure - Any
> > Sending email for trigger: Failure - Any
> >
> >
> >
> > ---------------------------------------------------------------------
> > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> > For additional commands, e-mail: dev-help@lucene.apache.org
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
>

Re: [JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 248 - Still Failing!

Posted by Steve Rowe <sa...@gmail.com>.
Hi David,

Looks like you meant to move the Bug Fixes section in Lucene’s CHANGES.txt down below the New Features section, but you left the original at the top when you created the new one?

--
Steve
www.lucidworks.com

> On Jul 6, 2016, at 4:17 PM, Policeman Jenkins Server <je...@thetaphi.de> wrote:
> 
> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/248/
> Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
> 
> All tests passed
> 
> Build Log:
> [...truncated 53916 lines...]
> changes-to-html:
>    [mkdir] Created dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/docs/changes
>      [get] Getting: https://issues.apache.org/jira/rest/api/2/project/LUCENE
>      [get] To: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/docs/changes/jiraVersionList.json
>     [exec] Section 'Bug Fixes' appears more than once under release '6.2.0' at /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/site/changes/changes2html.pl line 135.
> 
> BUILD FAILED
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:740: The following error occurred while executing this line:
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:101: The following error occurred while executing this line:
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build.xml:138: The following error occurred while executing this line:
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build.xml:479: The following error occurred while executing this line:
> /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:2498: exec returned: 255
> 
> Total time: 95 minutes 20 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> [WARNINGS] Skipping publisher since build result is FAILURE
> Recording test results
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 248 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/248/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

All tests passed

Build Log:
[...truncated 53916 lines...]
changes-to-html:
    [mkdir] Created dir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/docs/changes
      [get] Getting: https://issues.apache.org/jira/rest/api/2/project/LUCENE
      [get] To: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build/docs/changes/jiraVersionList.json
     [exec] Section 'Bug Fixes' appears more than once under release '6.2.0' at /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/site/changes/changes2html.pl line 135.

BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:740: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:101: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build.xml:138: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/build.xml:479: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:2498: exec returned: 255

Total time: 95 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 247 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/247/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test

Error Message:
Error from server at https://127.0.0.1:62422/solr: 'location' is not specified as a query parameter or as a default repository property or as a cluster property.

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:62422/solr: 'location' is not specified as a query parameter or as a default repository property or as a cluster property.
	at __randomizedtesting.SeedInfo.seed([DF0FC77DADC84BA4:575BF8A70334265C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:590)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:403)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:356)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1228)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:998)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:934)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testInvalidPath(AbstractCloudBackupRestoreTestCase.java:149)
	at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:128)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.overseer.ZkStateWriterTest.testExternalModificationToStateFormat2

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([DF0FC77DADC84BA4:AED2D3A9CC63B724]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.overseer.ZkStateWriterTest.testExternalModificationToStateFormat2(ZkStateWriterTest.java:322)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
searcher529 wasn't soon enough after soft529: Took 494ms, >= acceptable 300ms (fudge)

Stack Trace:
java.lang.AssertionError: searcher529 wasn't soon enough after soft529: Took 494ms, >= acceptable 300ms (fudge)
	at __randomizedtesting.SeedInfo.seed([DF0FC77DADC84BA4:8EDB3EFD1CBB7B03]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:140)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10683 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestLocalFSCloudBackupRestore
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/init-core-data-001
   [junit4]   2> 126875 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 126878 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 126878 INFO  (Thread-396) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 126878 INFO  (Thread-396) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 126978 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:37900
   [junit4]   2> 126978 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 126979 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 126989 INFO  (zkCallback-137-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@74d42ae2 name:ZooKeeperConnection Watcher:127.0.0.1:37900 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 126989 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 126989 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 126989 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 126993 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/clusterprops.json
   [junit4]   2> 127000 INFO  (jetty-launcher-136-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 127002 INFO  (jetty-launcher-136-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1ec49f98{/solr,null,AVAILABLE}
   [junit4]   2> 127005 INFO  (jetty-launcher-136-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@36c7c921{SSL,[ssl, http/1.1]}{127.0.0.1:60610}
   [junit4]   2> 127005 INFO  (jetty-launcher-136-thread-1) [    ] o.e.j.s.Server Started @129971ms
   [junit4]   2> 127005 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60610}
   [junit4]   2> 127005 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@6d06d69c
   [junit4]   2> 127005 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node1'
   [junit4]   2> 127006 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 127006 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 127006 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 127007 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 127014 INFO  (jetty-launcher-136-thread-2) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 127019 INFO  (jetty-launcher-136-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@56b34ffa{/solr,null,AVAILABLE}
   [junit4]   2> 127021 INFO  (jetty-launcher-136-thread-2) [    ] o.e.j.s.ServerConnector Started ServerConnector@7fc38960{SSL,[ssl, http/1.1]}{127.0.0.1:62422}
   [junit4]   2> 127021 INFO  (jetty-launcher-136-thread-2) [    ] o.e.j.s.Server Started @129987ms
   [junit4]   2> 127021 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62422}
   [junit4]   2> 127022 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@6d06d69c
   [junit4]   2> 127022 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2'
   [junit4]   2> 127022 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 127022 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 127022 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 127022 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 127028 INFO  (zkCallback-139-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@195fd839 name:ZooKeeperConnection Watcher:127.0.0.1:37900/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127028 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 127028 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 127028 INFO  (zkCallback-138-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4988de0 name:ZooKeeperConnection Watcher:127.0.0.1:37900/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127031 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 127031 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 127035 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 127035 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 127045 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/.
   [junit4]   2> 127045 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.CoreContainer New CoreContainer 1979577901
   [junit4]   2> 127045 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2]
   [junit4]   2> 127045 WARN  (jetty-launcher-136-thread-2) [    ] o.a.s.c.CoreContainer Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/lib to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/lib
   [junit4]   2> 127047 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : https,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 127047 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x155bf9ae7870001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 127048 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 127048 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x155bf9ae7870002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 127048 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node1/.
   [junit4]   2> 127048 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.CoreContainer New CoreContainer 1348828836
   [junit4]   2> 127048 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 127048 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node1]
   [junit4]   2> 127048 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 127048 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37900/solr
   [junit4]   2> 127048 WARN  (jetty-launcher-136-thread-1) [    ] o.a.s.c.CoreContainer Couldn't add files from /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node1/lib to classpath: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node1/lib
   [junit4]   2> 127048 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 127049 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : https,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,useRetries : false,
   [junit4]   2> 127049 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 127050 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 127050 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 127050 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 127050 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 127050 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:37900/solr
   [junit4]   2> 127050 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 127050 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 127051 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 127054 INFO  (zkCallback-142-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5bc98ef9 name:ZooKeeperConnection Watcher:127.0.0.1:37900 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127057 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 127057 INFO  (jetty-launcher-136-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 127059 INFO  (zkCallback-145-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3b2b92cb name:ZooKeeperConnection Watcher:127.0.0.1:37900 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127060 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$30/150612738@35859ece
   [junit4]   2> 127061 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 127062 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 127062 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x155bf9ae7870003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 127062 INFO  (jetty-launcher-136-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 127072 INFO  (zkCallback-146-thread-1-processing-n:127.0.0.1:62422_solr) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1edcf746 name:ZooKeeperConnection Watcher:127.0.0.1:37900/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127073 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 127074 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x155bf9ae7870004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 127075 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$30/150612738@7632edd1
   [junit4]   2> 127075 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 127079 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 127080 INFO  (zkCallback-147-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1acab36c name:ZooKeeperConnection Watcher:127.0.0.1:37900/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127080 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 127086 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 127087 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 127093 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 127094 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 127099 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 127103 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 127103 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 127106 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 127108 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 127111 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 127113 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 127116 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 127118 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 127120 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 127120 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 127124 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 127125 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 127128 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 127129 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96193639088586758-127.0.0.1:60610_solr-n_0000000000
   [junit4]   2> 127129 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60610_solr
   [junit4]   2> 127129 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 127131 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 127131 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer Overseer (id=96193639088586758-127.0.0.1:60610_solr-n_0000000000) starting
   [junit4]   2> 127133 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 127137 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 127141 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 127141 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 127141 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60610_solr
   [junit4]   2> 127142 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:60610_solr
   [junit4]   2> 127143 INFO  (zkCallback-147-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 127143 INFO  (zkCallback-146-thread-1-processing-n:127.0.0.1:62422_solr) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 127145 INFO  (zkCallback-147-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 127146 INFO  (OverseerCollectionConfigSetProcessor-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 127149 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 127150 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96193639088586757-127.0.0.1:62422_solr-n_0000000001
   [junit4]   2> 127152 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/96193639088586758-127.0.0.1:60610_solr-n_0000000000 to know if I could be the leader
   [junit4]   2> 127153 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:62422_solr as DOWN
   [junit4]   2> 127154 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62422_solr
   [junit4]   2> 127154 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:62422_solr
   [junit4]   2> 127156 INFO  (zkCallback-147-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 127159 INFO  (zkCallback-146-thread-2-processing-n:127.0.0.1:62422_solr) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 127165 INFO  (zkCallback-146-thread-2-processing-n:127.0.0.1:62422_solr) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 127169 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 127184 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 127188 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 127188 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 127189 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node1/.
   [junit4]   2> 127189 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 127192 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 127192 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 127193 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/.
   [junit4]   2> 127193 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 127194 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0
   [junit4]   2> 127194 INFO  (jetty-launcher-136-thread-2) [n:127.0.0.1:62422_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 127195 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0
   [junit4]   2> 127195 INFO  (jetty-launcher-136-thread-1) [n:127.0.0.1:60610_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 127195 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 127196 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 127199 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:62422_solr"} current state version: 0
   [junit4]   2> 127199 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:62422_solr
   [junit4]   2> 127201 INFO  (zkCallback-150-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1c114181 name:ZooKeeperConnection Watcher:127.0.0.1:37900 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127201 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 127201 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 127210 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x155bf9ae7870007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 127215 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 127215 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 127220 INFO  (zkCallback-153-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3f0f02de name:ZooKeeperConnection Watcher:127.0.0.1:37900/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127221 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 127221 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 127221 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 127227 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 127227 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 127237 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 127242 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[DF0FC77DADC84BA4]-worker) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 127314 INFO  (TEST-TestLocalFSCloudBackupRestore.test-seed#[DF0FC77DADC84BA4]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 127412 INFO  (qtp1494963009-901) [n:127.0.0.1:62422_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&property.customKey=customValue&collection.configName=conf1&maxShardsPerNode=2&autoAddReplicas=true&name=backuprestore&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 127432 INFO  (OverseerCollectionConfigSetProcessor-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2>   "name":"backuprestore",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "maxShardsPerNode":"2",
   [junit4]   2>   "autoAddReplicas":"true",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "property.customKey":"customValue",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 127433 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"backuprestore",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "maxShardsPerNode":"2",
   [junit4]   2>   "autoAddReplicas":"true",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "property.customKey":"customValue",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 127434 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerCollectionMessageHandler Create collection backuprestore
   [junit4]   2> 127440 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/backuprestore 
   [junit4]   2> 127440 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/backuprestore
   [junit4]   2> 127444 INFO  (zkCallback-153-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 127445 INFO  (zkCallback-147-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 127445 INFO  (zkCallback-146-thread-1-processing-n:127.0.0.1:62422_solr) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 127446 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"backuprestore",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "collection.configName":"conf1",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "maxShardsPerNode":"2",
   [junit4]   2>   "autoAddReplicas":"true",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "property.customKey":"customValue",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 127446 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: backuprestore
   [junit4]   2> 127449 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/backuprestore/state.json
   [junit4]   2> 127551 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerCollectionMessageHandler Creating SolrCores for new collection backuprestore, shardNames [shard1, shard2] , replicationFactor : 2
   [junit4]   2> 127553 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerCollectionMessageHandler Creating core backuprestore_shard2_replica2 as part of shard shard2 of collection backuprestore on 127.0.0.1:62422_solr
   [junit4]   2> 127556 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerCollectionMessageHandler Creating core backuprestore_shard1_replica2 as part of shard shard1 of collection backuprestore on 127.0.0.1:62422_solr
   [junit4]   2> 127566 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerCollectionMessageHandler Creating core backuprestore_shard2_replica1 as part of shard shard2 of collection backuprestore on 127.0.0.1:60610_solr
   [junit4]   2> 127566 INFO  (OverseerThreadFactory-980-thread-1-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.OverseerCollectionMessageHandler Creating core backuprestore_shard1_replica1 as part of shard shard1 of collection backuprestore on 127.0.0.1:60610_solr
   [junit4]   2> 127663 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr    ] o.a.s.h.a.CoreAdminOperation core create command property.customKey=customValue&qt=/admin/cores&collection.configName=conf1&name=backuprestore_shard1_replica2&action=CREATE&numShards=2&collection=backuprestore&shard=shard1&wt=javabin&version=2
   [junit4]   2> 127664 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr    ] o.a.s.h.a.CoreAdminOperation core create command property.customKey=customValue&qt=/admin/cores&collection.configName=conf1&name=backuprestore_shard2_replica2&action=CREATE&numShards=2&collection=backuprestore&shard=shard2&wt=javabin&version=2
   [junit4]   2> 127688 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=backuprestore_shard2_replica2, collection.configName=conf1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=backuprestore, customKey=customValue, numShards=2, dataDir=data/, shard=shard2}
   [junit4]   2> 127689 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=backuprestore_shard1_replica2, collection.configName=conf1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=backuprestore, customKey=customValue, numShards=2, dataDir=data/, shard=shard1}
   [junit4]   2> 127711 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 127712 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.ZkController Registering watch for collection backuprestore
   [junit4]   2> 127717 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"backuprestore_shard2_replica2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:62422/solr",
   [junit4]   2>   "node_name":"127.0.0.1:62422_solr",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 127721 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
   [junit4]   2>   "core":"backuprestore_shard2_replica2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:62422/solr",
   [junit4]   2>   "node_name":"127.0.0.1:62422_solr",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 127725 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 127729 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"backuprestore_shard1_replica2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:62422/solr",
   [junit4]   2>   "node_name":"127.0.0.1:62422_solr",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 127729 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
   [junit4]   2>   "core":"backuprestore_shard1_replica2",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:62422/solr",
   [junit4]   2>   "node_name":"127.0.0.1:62422_solr",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 127732 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.ZkController Registering watch for collection backuprestore
   [junit4]   2> 127733 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 127734 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.c.ZkStateReader Add data for [backuprestore] ver [0]
   [junit4]   2> 127735 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr    ] o.a.s.h.a.CoreAdminOperation core create command property.customKey=customValue&qt=/admin/cores&collection.configName=conf1&name=backuprestore_shard1_replica1&action=CREATE&numShards=2&collection=backuprestore&shard=shard1&wt=javabin&version=2
   [junit4]   2> 127735 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=backuprestore_shard1_replica1, collection.configName=conf1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=backuprestore, customKey=customValue, numShards=2, dataDir=data/, shard=shard1}
   [junit4]   2> 127736 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 127737 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.ZkController Registering watch for collection backuprestore
   [junit4]   2> 127738 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"backuprestore_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:60610/solr",
   [junit4]   2>   "node_name":"127.0.0.1:60610_solr",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 127738 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
   [junit4]   2>   "core":"backuprestore_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:60610/solr",
   [junit4]   2>   "node_name":"127.0.0.1:60610_solr",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 127739 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.c.ZkStateReader Add data for [backuprestore] ver [0]
   [junit4]   2> 127741 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 127742 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 127742 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr    ] o.a.s.h.a.CoreAdminOperation core create command property.customKey=customValue&qt=/admin/cores&collection.configName=conf1&name=backuprestore_shard2_replica1&action=CREATE&numShards=2&collection=backuprestore&shard=shard2&wt=javabin&version=2
   [junit4]   2> 127743 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=backuprestore_shard2_replica1, collection.configName=conf1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=backuprestore, customKey=customValue, numShards=2, dataDir=data/, shard=shard2}
   [junit4]   2> 127743 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 127745 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.ZkController Registering watch for collection backuprestore
   [junit4]   2> 127745 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 127747 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"backuprestore_shard2_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:60610/solr",
   [junit4]   2>   "node_name":"127.0.0.1:60610_solr",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 127747 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
   [junit4]   2>   "core":"backuprestore_shard2_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:60610/solr",
   [junit4]   2>   "node_name":"127.0.0.1:60610_solr",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "collection":"backuprestore",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 127882 INFO  (OverseerStateUpdate-96193639088586758-127.0.0.1:60610_solr-n_0000000000) [n:127.0.0.1:60610_solr    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/backuprestore/state.json version: 0
   [junit4]   2> 127885 INFO  (zkCallback-146-thread-1-processing-n:127.0.0.1:62422_solr) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 127885 INFO  (zkCallback-147-thread-3-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore/state.json] for collection [backuprestore] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 127886 INFO  (zkCallback-146-thread-1-processing-n:127.0.0.1:62422_solr) [n:127.0.0.1:62422_solr    ] o.a.s.c.c.ZkStateReader Updating data for [backuprestore] from [0] to [1]
   [junit4]   2> 127886 INFO  (zkCallback-147-thread-3-processing-n:127.0.0.1:60610_solr) [n:127.0.0.1:60610_solr    ] o.a.s.c.c.ZkStateReader Updating data for [backuprestore] from [0] to [1]
   [junit4]   2> 128742 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.ZkController waiting to find shard id in clusterstate for backuprestore_shard1_replica1
   [junit4]   2> 128742 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.ZkController Check for collection zkNode:backuprestore
   [junit4]   2> 128743 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.ZkController waiting to find shard id in clusterstate for backuprestore_shard2_replica2
   [junit4]   2> 128743 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.ZkController Check for collection zkNode:backuprestore
   [junit4]   2> 128743 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 128743 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/backuprestore]
   [junit4]   2> 128744 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.c.ZkStateReader path=[/collections/backuprestore] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 128745 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node1/backuprestore_shard1_replica1'
   [junit4]   2> 128745 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 128745 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 128745 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.ZkController waiting to find shard id in clusterstate for backuprestore_shard1_replica2
   [junit4]   2> 128745 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.ZkController Check for collection zkNode:backuprestore
   [junit4]   2> 128748 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 128751 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.ZkController waiting to find shard id in clusterstate for backuprestore_shard2_replica1
   [junit4]   2> 128751 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.ZkController Check for collection zkNode:backuprestore
   [junit4]   2> 128751 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 128751 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/backuprestore]
   [junit4]   2> 128752 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.c.ZkStateReader path=[/collections/backuprestore] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 128752 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node1/backuprestore_shard2_replica1'
   [junit4]   2> 128752 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 128752 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 128753 INFO  (qtp1864397008-888) [n:127.0.0.1:60610_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 128759 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 128760 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.2.0
   [junit4]   2> 128766 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 128766 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/backuprestore]
   [junit4]   2> 128767 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.c.ZkStateReader path=[/collections/backuprestore] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 128767 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/backuprestore_shard2_replica2'
   [junit4]   2> 128767 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 128767 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 128768 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 128768 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/backuprestore]
   [junit4]   2> 128769 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.c.ZkStateReader path=[/collections/backuprestore] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 128769 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/backuprestore_shard1_replica2'
   [junit4]   2> 128769 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 128769 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 128805 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 128807 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 128809 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 128820 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 128822 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.s.IndexSchema [backuprestore_shard1_replica1] Schema name=minimal
   [junit4]   2> 128822 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.2.0
   [junit4]   2> 128825 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 128827 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.2.0
   [junit4]   2> 128827 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 128830 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 128831 INFO  (qtp1864397008-890) [n:127.0.0.1:60610_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 128835 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.s.IndexSchema [backuprestore_shard2_replica2] Schema name=minimal
   [junit4]   2> 128835 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.s.IndexSchema [backuprestore_shard1_replica2] Schema name=minimal
   [junit4]   2> 128837 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 128838 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 128839 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0
   [junit4]   2> 128839 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2  x:backuprestore_shard2_replica2] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard2_replica2' using configuration from collection backuprestore
   [junit4]   2> 128840 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 128840 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrCore [[backuprestore_shard2_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/backuprestore_shard2_replica2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard2_replica2/data/]
   [junit4]   2> 128840 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrCore JMX monitoring not detected for core: backuprestore_shard2_replica2
   [junit4]   2> 128840 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.CachingDirectoryFactory return new directory for /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard2_replica2/data
   [junit4]   2> 128841 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrCore New index directory detected: old=null new=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard2_replica2/data/index/
   [junit4]   2> 128841 WARN  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrCore [backuprestore_shard2_replica2] Solr index directory '/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 128844 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.CachingDirectoryFactory return new directory for /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard2_replica2/data/index
   [junit4]   2> 128845 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1f23fa6e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2a75aeb2),segFN=segments_1,generation=1}
   [junit4]   2> 128845 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 128845 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0
   [junit4]   2> 128845 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1  x:backuprestore_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_shard1_replica2' using configuration from collection backuprestore
   [junit4]   2> 128845 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 128845 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrCore [[backuprestore_shard1_replica2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/backuprestore_shard1_replica2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard1_replica2/data/]
   [junit4]   2> 128845 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrCore JMX monitoring not detected for core: backuprestore_shard1_replica2
   [junit4]   2> 128846 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.CachingDirectoryFactory return new directory for /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard1_replica2/data
   [junit4]   2> 128846 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrCore New index directory detected: old=null new=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard1_replica2/data/index/
   [junit4]   2> 128846 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 128846 WARN  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrCore [backuprestore_shard1_replica2] Solr index directory '/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 128847 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.CachingDirectoryFactory return new directory for /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard1_replica2/data/index
   [junit4]   2> 128847 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@5e9e3e11 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b49dbcf),segFN=segments_1,generation=1}
   [junit4]   2> 128847 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 128848 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 128849 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 128849 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 128849 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 128851 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 128851 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 128852 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 128852 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 128852 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 128853 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 128853 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 128853 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 128854 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 128854 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 128855 INFO  (OldIndexDirectoryCleanupThreadForCore-backuprestore_shard2_replica2) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrCore Looking for old index directories to cleanup for core backuprestore_shard2_replica2 in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard2_replica2/data/
   [junit4]   2> 128855 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 128856 INFO  (OldIndexDirectoryCleanupThreadForCore-backuprestore_shard1_replica2) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrCore Looking for old index directories to cleanup for core backuprestore_shard1_replica2 in /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard1_replica2/data/
   [junit4]   2> 128856 WARN  (OldIndexDirectoryCleanupThreadForCore-backuprestore_shard1_replica2) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.DirectoryFactory /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestLocalFSCloudBackupRestore_DF0FC77DADC84BA4-001/tempDir-001/node2/./backuprestore_shard1_replica2/data/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 128856 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1f23fa6e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2a75aeb2),segFN=segments_1,generation=1}
   [junit4]   2> 128856 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 128857 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@c956627[backuprestore_shard2_replica2] main]
   [junit4]   2> 128857 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/backuprestore]
   [junit4]   2> 128858 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.c.c.ZkStateReader path=[/collections/backuprestore] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 128858 INFO  (qtp1494963009-900) [n:127.0.0.1:62422_solr c:backuprestore s:shard2 r:core_node1 x:backuprestore_shard2_replica2] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 128858 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,/update/csv,/sql,/update/json/docs,/admin/luke,/graph,/export,/select,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/stream,/schema,/admin/plugins,/admin/logging,/update/json,/admin/ping,/admin/threads,/update,/admin/file
   [junit4]   2> 128858 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 128858 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_node2 x:backuprestore_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 128858 INFO  (qtp1494963009-899) [n:127.0.0.1:62422_solr c:backuprestore s:shard1 r:core_nod

[...truncated too long message...]

335 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.u.DefaultSolrCoreState SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2377335 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.u.DefaultSolrCoreState closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2377335 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 2377339 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1defb932 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ca0d2bc),segFN=segments_4,generation=4}
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1defb932 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ca0d2bc),segFN=segments_5,generation=5}
   [junit4]   2> 2377339 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 5
   [junit4]   2> 2377340 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Closing main searcher on request.
   [junit4]   2> 2377353 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2377353 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_DF0FC77DADC84BA4-001/init-core-data-001/index [CachedDir<<refCount=0;path=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_DF0FC77DADC84BA4-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 2377353 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 2377353 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_DF0FC77DADC84BA4-001/init-core-data-001/index
   [junit4]   2> 2377353 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_DF0FC77DADC84BA4-001/init-core-data-001 [CachedDir<<refCount=0;path=/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_DF0FC77DADC84BA4-001/init-core-data-001;done=false>>]
   [junit4]   2> 2377354 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 2377354 INFO  (coreCloseExecutor-7964-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_DF0FC77DADC84BA4-001/init-core-data-001
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_DF0FC77DADC84BA4-001
   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=ClassicSimilarity, locale=ar-IQ, timezone=Australia/Canberra
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=3,threads=1,free=35549992,total=492830720
   [junit4]   2> NOTE: All tests run in this JVM: [TestCursorMarkWithoutUniqueKey, TestConfigSetsAPIZkFailure, TestReloadDeadlock, SmileWriterTest, SimplePostToolTest, TestReplicationHandlerBackup, JSONWriterTest, TestLazyCores, TestClassicSimilarityFactory, DistributedQueryElevationComponentTest, TestSurroundQueryParser, TestAnalyzedSuggestions, TestFaceting, IndexBasedSpellCheckerTest, TestStandardQParsers, TestCodecSupport, GraphQueryTest, SimpleFacetsTest, SliceStateTest, TestManagedStopFilterFactory, TestStressVersions, TestManagedSchema, TestSizeLimitedDistributedMap, XsltUpdateRequestHandlerTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestLocalFSCloudBackupRestore, DocValuesMissingTest, TestPostingsSolrHighlighter, TestExactSharedStatsCache, ZkStateWriterTest, TermsComponentTest, HttpPartitionTest, TestRebalanceLeaders, TestRTimerTree, TestConfigOverlay, TestCoreContainer, TestHashPartitioner, SchemaVersionSpecificBehaviorTest, ExplicitHLLTest, TestDefaultStatsCache, AnalysisErrorHandlingTest, TestConfigReload, TestSubQueryTransformerDistrib, CustomCollectionTest, TestNonDefinedSimilarityFactory, JavabinLoaderTest, TestPerFieldSimilarityWithDefaultOverride, PKIAuthenticationIntegrationTest, TestRequestForwarding, BadComponentTest, CheckHdfsIndexTest, HdfsWriteToMultipleCollectionsTest, DistributedFacetPivotSmallAdvancedTest, HdfsTlogReplayBufferedWhileIndexingTest, HLLSerializationTest, IndexSchemaTest, TestRemoteStreaming, TestRestoreCore, TestFreeTextSuggestions, MoreLikeThisHandlerTest, TestGroupingSearch, DirectSolrSpellCheckerTest, TestSchemaNameResource, SecurityConfHandlerTest, RestartWhileUpdatingTest, TestJsonFacets, DocValuesMultiTest, CloudMLTQParserTest, TestLRUStatsCache, TestSimpleQParserPlugin, TestUpdate, TestDistribDocBasedVersion, ShardSplitTest, TestFieldSortValues, AnalyticsQueryTest, URLClassifyProcessorTest, PreAnalyzedUpdateProcessorTest, DistributedFacetPivotLargeTest, QueryElevationComponentTest, TestSolrDeletionPolicy1, SolrCLIZkUtilsTest, DistributedFacetPivotWhiteBoxTest, TestQuerySenderNoQuery, TestStressReorder, DeleteShardTest, BasicDistributedZkTest, TestComponentsName, XmlUpdateRequestHandlerTest, TestOrdValues, OutOfBoxZkACLAndCredentialsProvidersTest, MergeStrategyTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, AtomicUpdatesTest, AlternateDirectoryTest, TestRawTransformer, SystemInfoHandlerTest, HdfsLockFactoryTest, VMParamsZkACLAndCredentialsProvidersTest, TestSearchPerf, TestFieldCacheSortRandom, TestJmxMonitoredMap, TestElisionMultitermQuery, PolyFieldTest, SSLMigrationTest, PrimUtilsTest, OverseerRolesTest, ZkCLITest, ChangedSchemaMergeTest, BlobRepositoryCloudTest, TlogReplayBufferedWhileIndexingTest, CoreMergeIndexesAdminHandlerTest, ConnectionManagerTest, TestTestInjection, LeaderElectionTest, SortSpecParsingTest, SolrIndexConfigTest, FacetPivotSmallTest, TestSubQueryTransformer, TestSlowCompositeReaderWrapper, OverseerTest, HdfsChaosMonkeySafeLeaderTest, TestSchemaSimilarityResource, TestClusterStateMutator, TestDistributedStatsComponentCardinality, PingRequestHandlerTest, ConjunctionSolrSpellCheckerTest, LeaderInitiatedRecoveryOnCommitTest, DataDrivenBlockJoinTest, RecoveryZkTest, MBeansHandlerTest, TestConfigSetsAPI, TestCloudManagedSchema, RequiredFieldsTest, RuleEngineTest, TestDistributedSearch, TestSortingResponseWriter, StatelessScriptUpdateProcessorFactoryTest, SolrCoreTest, TestSolr4Spatial2, BigEndianAscendingWordSerializerTest, TestExclusionRuleCollectionAccess, TestFieldTypeCollectionResource, TestMissingGroups, HdfsNNFailoverTest, TestCopyFieldCollectionResource, SolrXmlInZkTest, TestBlendedInfixSuggestions, ClusterStateTest, SuggesterFSTTest, SolrRequestParserTest, RemoteQueryErrorTest, DistribCursorPagingTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, ShardRoutingTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, ClusterStateUpdateTest, TestRandomFaceting, TestZkChroot, TestRandomDVFaceting, ShardRoutingCustomTest, TestRecovery, DistributedSpellCheckComponentTest, ZkControllerTest, TestRealTimeGet, TestJoin, TestReload, HardAutoCommitTest, TestRangeQuery, TestSolr4Spatial, SolrCmdDistributorTest, PeerSyncTest, ConvertedLegacyTest, TestSort, TestFiltering, TestFunctionQuery, BasicFunctionalityTest, DirectUpdateHandlerTest, TestBadConfig, TestIndexSearcher, HighlighterTest, SoftAutoCommitTest]
   [junit4] Completed [300/620 (3!)] on J0 in 7.29s, 3 tests, 1 failure <<< FAILURES!

[...truncated 1059 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:740: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:684: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:59: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build.xml:233: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/common-build.xml:530: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:1427: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:984: There were test failures: 620 suites (10 ignored), 2596 tests, 1 error, 2 failures, 86 ignored (71 assumptions) [seed: DF0FC77DADC84BA4]

Total time: 84 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 246 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/246/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy

Error Message:
expected:<2> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<2> but was:<1>
	at __randomizedtesting.SeedInfo.seed([8401079BE870F961:2BFA29BA4C5B405D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdate(ZkStateReaderTest.java:130)
	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy(ZkStateReaderTest.java:58)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10597 lines...]
   [junit4] Suite: org.apache.solr.cloud.overseer.ZkStateReaderTest
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.overseer.ZkStateReaderTest_8401079BE870F961-001/init-core-data-001
   [junit4]   2> 72798 INFO  (SUITE-ZkStateReaderTest-seed#[8401079BE870F961]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 72800 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithExplicitRefreshLazy
   [junit4]   2> 72800 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 72800 INFO  (Thread-43) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 72800 INFO  (Thread-43) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 72900 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer start zk server on port:37162
   [junit4]   2> 72902 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 72903 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 72909 INFO  (zkCallback-7386-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@27c8c7f5 name:ZooKeeperConnection Watcher:127.0.0.1:37162 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 72909 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 72911 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 72913 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 72914 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 72916 INFO  (zkCallback-7387-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6d2411d4 name:ZooKeeperConnection Watcher:127.0.0.1:37162 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 72917 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 72917 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 72917 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 72925 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 72925 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 72956 INFO  (zkCallback-7388-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@81fd303 name:ZooKeeperConnection Watcher:127.0.0.1:37162/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 72956 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 72956 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 72957 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 72959 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 72961 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 72963 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 72965 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 72967 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 72968 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 72969 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 72971 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 72973 INFO  (zkCallback-7388-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 72985 INFO  (zkCallback-7388-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 72989 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 72990 INFO  (zkCallback-7388-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 72996 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37162 37162
   [junit4]   2> 88034 INFO  (Thread-43) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37162 37162
   [junit4]   2> 88035 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefreshLazy-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithExplicitRefreshLazy
   [junit4]   2> 88038 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithTimeDelayLazy
   [junit4]   2> 88038 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 88038 INFO  (Thread-44) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 88038 INFO  (Thread-44) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 88138 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer start zk server on port:35573
   [junit4]   2> 88139 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 88139 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 88175 INFO  (zkCallback-7390-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@42043950 name:ZooKeeperConnection Watcher:127.0.0.1:35573 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 88175 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 88175 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 88178 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 88178 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 88182 INFO  (zkCallback-7391-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4e841783 name:ZooKeeperConnection Watcher:127.0.0.1:35573 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 88182 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 88182 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 88182 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 88189 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 88189 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 88191 INFO  (zkCallback-7392-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1122c965 name:ZooKeeperConnection Watcher:127.0.0.1:35573/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 88191 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 88225 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 88226 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 88229 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 88232 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 88234 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 88237 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 88238 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 88239 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 88242 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 88243 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 88258 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 88260 INFO  (zkCallback-7392-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 88262 INFO  (zkCallback-7392-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 88271 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 88276 INFO  (zkCallback-7392-thread-1) [    ] o.a.s.c.c.ZkStateReader Uninteresting collection c1
   [junit4]   2> 88277 INFO  (zkCallback-7392-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 88278 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 88281 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35573 35573
   [junit4]   2> 98307 INFO  (Thread-44) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35573 35573
   [junit4]   2> 98311 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithTimeDelayLazy
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ZkStateReaderTest -Dtests.method=testStateFormatUpdateWithTimeDelayLazy -Dtests.seed=8401079BE870F961 -Dtests.slow=true -Dtests.locale=zh-SG -Dtests.timezone=America/Glace_Bay -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 10.3s J0 | ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<2> but was:<1>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8401079BE870F961:2BFA29BA4C5B405D]:0)
   [junit4]    > 	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdate(ZkStateReaderTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.overseer.ZkStateReaderTest.testStateFormatUpdateWithTimeDelayLazy(ZkStateReaderTest.java:58)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 98317 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testExternalCollectionWatchedNotWatched
   [junit4]   2> 98317 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 98318 INFO  (Thread-45) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 98318 INFO  (Thread-45) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 98423 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer start zk server on port:52747
   [junit4]   2> 98423 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 98469 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 98472 INFO  (zkCallback-7394-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@180386ca name:ZooKeeperConnection Watcher:127.0.0.1:52747 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98472 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 98512 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 98515 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 98515 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 98517 INFO  (zkCallback-7395-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@70552100 name:ZooKeeperConnection Watcher:127.0.0.1:52747 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98517 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 98518 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 98518 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 98523 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 98523 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 98526 INFO  (zkCallback-7396-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@756cd1a1 name:ZooKeeperConnection Watcher:127.0.0.1:52747/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98526 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 98527 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 98528 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 98531 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 98533 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 98536 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 98539 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 98540 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 98542 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 98543 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 98546 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 98549 INFO  (zkCallback-7396-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 98552 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 98556 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 98558 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x155bdb206e90002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 98634 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52747 52747
   [junit4]   2> 103754 INFO  (Thread-45) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:52747 52747
   [junit4]   2> 103757 INFO  (TEST-ZkStateReaderTest.testExternalCollectionWatchedNotWatched-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testExternalCollectionWatchedNotWatched
   [junit4]   2> 103761 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithExplicitRefresh
   [junit4]   2> 103763 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 103763 INFO  (Thread-46) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 103763 INFO  (Thread-46) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 103863 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer start zk server on port:46093
   [junit4]   2> 103863 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 103863 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 103909 INFO  (zkCallback-7398-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3ffecb85 name:ZooKeeperConnection Watcher:127.0.0.1:46093 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103909 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 103910 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 103912 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 103913 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 103917 INFO  (zkCallback-7399-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@50e225f6 name:ZooKeeperConnection Watcher:127.0.0.1:46093 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103917 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 103917 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 103917 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 103923 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 103924 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 103966 INFO  (zkCallback-7400-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@44287cfa name:ZooKeeperConnection Watcher:127.0.0.1:46093/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103966 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 103967 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 103967 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 103971 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 103975 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 103978 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 103981 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 103983 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 103984 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 103985 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 103989 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 103990 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 103996 INFO  (zkCallback-7400-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 103999 INFO  (zkCallback-7400-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 104001 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 104003 INFO  (zkCallback-7400-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 104004 INFO  (zkCallback-7400-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 104004 INFO  (zkCallback-7400-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 104010 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46093 46093
   [junit4]   2> 109029 INFO  (Thread-46) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46093 46093
   [junit4]   2> 109031 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithExplicitRefresh-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithExplicitRefresh
   [junit4]   2> 109033 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testWatchedCollectionCreation
   [junit4]   2> 109033 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 109034 INFO  (Thread-47) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 109034 INFO  (Thread-47) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 109134 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer start zk server on port:49772
   [junit4]   2> 109134 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 109135 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 109172 INFO  (zkCallback-7402-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3a4a39db name:ZooKeeperConnection Watcher:127.0.0.1:49772 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 109172 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 109172 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 109175 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 109176 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 109177 INFO  (zkCallback-7403-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6ecf0afd name:ZooKeeperConnection Watcher:127.0.0.1:49772 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 109177 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 109178 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 109178 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 109181 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 109182 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 109246 INFO  (zkCallback-7404-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3fa289dc name:ZooKeeperConnection Watcher:127.0.0.1:49772/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 109247 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 109247 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 109247 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 109251 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 109253 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 109256 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 109259 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 109260 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 109262 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 109263 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 109265 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 109266 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 109269 INFO  (zkCallback-7404-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 109271 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 109272 INFO  (zkCallback-7404-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 109273 INFO  (zkCallback-7404-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 109275 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x155bdb230c40002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 109276 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49772 49772
   [junit4]   2> 119299 INFO  (Thread-47) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49772 49772
   [junit4]   2> 119302 INFO  (TEST-ZkStateReaderTest.testWatchedCollectionCreation-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testWatchedCollectionCreation
   [junit4]   2> 119305 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testStateFormatUpdateWithTimeDelay
   [junit4]   2> 119306 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 119306 INFO  (Thread-48) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 119306 INFO  (Thread-48) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 119406 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer start zk server on port:59273
   [junit4]   2> 119407 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 119408 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 119416 INFO  (zkCallback-7406-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1c42e220 name:ZooKeeperConnection Watcher:127.0.0.1:59273 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 119416 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 119416 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 119421 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 119422 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 119424 INFO  (zkCallback-7407-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3205e90 name:ZooKeeperConnection Watcher:127.0.0.1:59273 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 119424 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 119424 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 119424 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 119431 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 119432 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 119480 INFO  (zkCallback-7408-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@417629ec name:ZooKeeperConnection Watcher:127.0.0.1:59273/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 119480 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 119480 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 119481 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 119484 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 119485 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 119487 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 119489 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 119490 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 119491 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 119492 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 119495 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.ZkStateReader Deleting data for [c1]
   [junit4]   2> 119495 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/c1
   [junit4]   2> 119511 INFO  (zkCallback-7408-thread-1) [    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 119515 INFO  (zkCallback-7408-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 119517 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/c1/state.json
   [junit4]   2> 119519 INFO  (zkCallback-7408-thread-1) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeCreated path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [0])
   [junit4]   2> 119520 INFO  (zkCallback-7408-thread-1) [    ] o.a.s.c.c.ZkStateReader Add data for [c1] ver [0]
   [junit4]   2> 119552 INFO  (zkCallback-7408-thread-2) [    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 119556 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59273 59273
   [junit4]   2> 119570 INFO  (Thread-48) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59273 59273
   [junit4]   2> 119575 INFO  (TEST-ZkStateReaderTest.testStateFormatUpdateWithTimeDelay-seed#[8401079BE870F961]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testStateFormatUpdateWithTimeDelay
   [junit4]   2> 119576 INFO  (SUITE-ZkStateReaderTest-seed#[8401079BE870F961]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.overseer.ZkStateReaderTest_8401079BE870F961-001
   [junit4]   2> Jul 06, 2016 12:53:50 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=1148, maxMBSortInHeap=6.582266883148929, sim=ClassicSimilarity, locale=zh-SG, timezone=America/Glace_Bay
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_92 (64-bit)/cpus=3,threads=1,free=134919712,total=214958080
   [junit4]   2> NOTE: All tests run in this JVM: [SchemaVersionSpecificBehaviorTest, TestStressLiveNodes, SolrTestCaseJ4Test, HLLUtilTest, HdfsRestartWhileUpdatingTest, SuggesterFSTTest, ZkStateReaderTest]
   [junit4] Completed [21/619 (1!)] on J0 in 47.85s, 6 tests, 1 failure <<< FAILURES!

[...truncated 1945 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:740: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:684: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:59: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build.xml:233: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/common-build.xml:530: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:1427: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:984: There were test failures: 619 suites (10 ignored), 2595 tests, 1 failure, 86 ignored (71 assumptions) [seed: 8401079BE870F961]

Total time: 84 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-6.x-Solaris (64bit/jdk1.8.0) - Build # 245 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-6.x-Solaris/245/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cloud.TestOnReconnectListenerSupport.test

Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:39405 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:39405 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([8008AAF331A6914F:85C95299F5AFCB7]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:116)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:98)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.printLayout(AbstractDistribZkTestBase.java:295)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.distribTearDown(AbstractFullDistribZkTestBase.java:1500)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:969)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:39405 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:228)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
	... 37 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestOnReconnectListenerSupport

Error Message:
54 threads leaked from SUITE scope at org.apache.solr.cloud.TestOnReconnectListenerSupport:     1) Thread[id=4045, name=qtp1303295522-4045, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=4073, name=qtp599160364-4073, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=4021, name=TEST-TestOnReconnectListenerSupport.test-seed#[8008AAF331A6914F]-EventThread, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    4) Thread[id=4056, name=qtp1133684582-4056, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=4027, name=Thread-1464, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1247)    6) Thread[id=4070, name=qtp599160364-4070, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=4008, name=qtp172085939-4008, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=4047, name=Scheduler-204361560, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=4057, name=qtp1133684582-4057, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=4075, name=Scheduler-1455710535, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=4032, name=searcherExecutor-1740-thread-1, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=4051, name=Scheduler-948630609, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=4059, name=qtp1133684582-4059, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=4009, name=qtp172085939-4009, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=4061, name=org.eclipse.jetty.server.session.HashSessionManager@6550a8fbTimer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=4052, name=qtp1133684582-4052-acceptor-0@5366db8b-ServerConnector@6925eabd{HTTP/1.1,[http/1.1]}{127.0.0.1:45739}, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   17) Thread[id=4013, name=org.eclipse.jetty.server.session.HashSessionManager@4fa93b86Timer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   18) Thread[id=4046, name=qtp1303295522-4046, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   19) Thread[id=4005, name=qtp172085939-4005, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   20) Thread[id=4004, name=qtp172085939-4004-acceptor-0@e84c710-ServerConnector@206722ca{HTTP/1.1,[http/1.1]}{127.0.0.1:62786}, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   21) Thread[id=4020, name=TEST-TestOnReconnectListenerSupport.test-seed#[8008AAF331A6914F]-SendThread(127.0.0.1:39405), state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)   22) Thread[id=4071, name=qtp599160364-4071, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   23) Thread[id=3997, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)   24) Thread[id=3995, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   25) Thread[id=4042, name=qtp1303295522-4042, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   26) Thread[id=4023, name=OverseerStateUpdate-96189400141201412-127.0.0.1:62786_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)         at java.lang.Thread.run(Thread.java:745)   27) Thread[id=4039, name=qtp1303295522-4039, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   28) Thread[id=4068, name=qtp599160364-4068-acceptor-0@d6cff6d-ServerConnector@799e577f{HTTP/1.1,[http/1.1]}{127.0.0.1:36986}, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   29) Thread[id=4007, name=qtp172085939-4007, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   30) Thread[id=4076, name=org.eclipse.jetty.server.session.HashSessionManager@3380d841Timer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   31) Thread[id=4011, name=qtp172085939-4011, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   32) Thread[id=4040, name=qtp1303295522-4040-acceptor-0@f78940f-ServerConnector@317873b2{HTTP/1.1,[http/1.1]}{127.0.0.1:39405}, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   33) Thread[id=4036, name=TEST-TestOnReconnectListenerSupport.test-seed#[8008AAF331A6914F]-SendThread(127.0.0.1:39405), state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)   34) Thread[id=4069, name=qtp599160364-4069, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   35) Thread[id=3993, name=Thread-1455, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1245)         at java.lang.Thread.join(Thread.java:1319)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491)   36) Thread[id=3994, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:745)   37) Thread[id=4043, name=qtp1303295522-4043, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   38) Thread[id=4041, name=qtp1303295522-4041, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   39) Thread[id=4037, name=TEST-TestOnReconnectListenerSupport.test-seed#[8008AAF331A6914F]-EventThread, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   40) Thread[id=4053, name=qtp1133684582-4053, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   41) Thread[id=4054, name=qtp1133684582-4054, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   42) Thread[id=4062, name=qtp1133684582-4062, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   43) Thread[id=4072, name=qtp599160364-4072, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   44) Thread[id=4060, name=Scheduler-1907355904, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   45) Thread[id=4006, name=qtp172085939-4006, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   46) Thread[id=4012, name=Scheduler-1900208098, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   47) Thread[id=4048, name=org.eclipse.jetty.server.session.HashSessionManager@21c67c61Timer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   48) Thread[id=4025, name=OverseerHdfsCoreFailoverThread-96189400141201412-127.0.0.1:62786_-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)   49) Thread[id=4044, name=qtp1303295522-4044, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   50) Thread[id=4058, name=qtp1133684582-4058, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   51) Thread[id=4067, name=qtp599160364-4067, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]         at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)         at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   52) Thread[id=4010, name=qtp172085939-4010, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   53) Thread[id=4074, name=qtp599160364-4074, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   54) Thread[id=4024, name=OverseerCollectionConfigSetProcessor-96189400141201412-127.0.0.1:62786_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:343)         at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:224)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 54 threads leaked from SUITE scope at org.apache.solr.cloud.TestOnReconnectListenerSupport: 
   1) Thread[id=4045, name=qtp1303295522-4045, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=4073, name=qtp599160364-4073, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=4021, name=TEST-TestOnReconnectListenerSupport.test-seed#[8008AAF331A6914F]-EventThread, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   4) Thread[id=4056, name=qtp1133684582-4056, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=4027, name=Thread-1464, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1247)
   6) Thread[id=4070, name=qtp599160364-4070, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=4008, name=qtp172085939-4008, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=4047, name=Scheduler-204361560, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=4057, name=qtp1133684582-4057, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=4075, name=Scheduler-1455710535, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=4032, name=searcherExecutor-1740-thread-1, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=4051, name=Scheduler-948630609, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=4059, name=qtp1133684582-4059, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=4009, name=qtp172085939-4009, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=4061, name=org.eclipse.jetty.server.session.HashSessionManager@6550a8fbTimer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=4052, name=qtp1133684582-4052-acceptor-0@5366db8b-ServerConnector@6925eabd{HTTP/1.1,[http/1.1]}{127.0.0.1:45739}, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  17) Thread[id=4013, name=org.eclipse.jetty.server.session.HashSessionManager@4fa93b86Timer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  18) Thread[id=4046, name=qtp1303295522-4046, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  19) Thread[id=4005, name=qtp172085939-4005, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  20) Thread[id=4004, name=qtp172085939-4004-acceptor-0@e84c710-ServerConnector@206722ca{HTTP/1.1,[http/1.1]}{127.0.0.1:62786}, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  21) Thread[id=4020, name=TEST-TestOnReconnectListenerSupport.test-seed#[8008AAF331A6914F]-SendThread(127.0.0.1:39405), state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  22) Thread[id=4071, name=qtp599160364-4071, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  23) Thread[id=3997, name=ProcessThread(sid:0 cport:-1):, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
  24) Thread[id=3995, name=SessionTracker, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at java.lang.Object.wait(Native Method)
        at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
  25) Thread[id=4042, name=qtp1303295522-4042, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  26) Thread[id=4023, name=OverseerStateUpdate-96189400141201412-127.0.0.1:62786_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:196)
        at java.lang.Thread.run(Thread.java:745)
  27) Thread[id=4039, name=qtp1303295522-4039, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  28) Thread[id=4068, name=qtp599160364-4068-acceptor-0@d6cff6d-ServerConnector@799e577f{HTTP/1.1,[http/1.1]}{127.0.0.1:36986}, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  29) Thread[id=4007, name=qtp172085939-4007, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  30) Thread[id=4076, name=org.eclipse.jetty.server.session.HashSessionManager@3380d841Timer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  31) Thread[id=4011, name=qtp172085939-4011, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  32) Thread[id=4040, name=qtp1303295522-4040-acceptor-0@f78940f-ServerConnector@317873b2{HTTP/1.1,[http/1.1]}{127.0.0.1:39405}, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  33) Thread[id=4036, name=TEST-TestOnReconnectListenerSupport.test-seed#[8008AAF331A6914F]-SendThread(127.0.0.1:39405), state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  34) Thread[id=4069, name=qtp599160364-4069, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  35) Thread[id=3993, name=Thread-1455, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1245)
        at java.lang.Thread.join(Thread.java:1319)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:302)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491)
  36) Thread[id=3994, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
        at java.lang.Thread.run(Thread.java:745)
  37) Thread[id=4043, name=qtp1303295522-4043, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  38) Thread[id=4041, name=qtp1303295522-4041, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  39) Thread[id=4037, name=TEST-TestOnReconnectListenerSupport.test-seed#[8008AAF331A6914F]-EventThread, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  40) Thread[id=4053, name=qtp1133684582-4053, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  41) Thread[id=4054, name=qtp1133684582-4054, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  42) Thread[id=4062, name=qtp1133684582-4062, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  43) Thread[id=4072, name=qtp599160364-4072, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  44) Thread[id=4060, name=Scheduler-1907355904, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  45) Thread[id=4006, name=qtp172085939-4006, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  46) Thread[id=4012, name=Scheduler-1900208098, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  47) Thread[id=4048, name=org.eclipse.jetty.server.session.HashSessionManager@21c67c61Timer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  48) Thread[id=4025, name=OverseerHdfsCoreFailoverThread-96189400141201412-127.0.0.1:62786_-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
  49) Thread[id=4044, name=qtp1303295522-4044, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  50) Thread[id=4058, name=qtp1133684582-4058, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  51) Thread[id=4067, name=qtp599160364-4067, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
        at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
        at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  52) Thread[id=4010, name=qtp172085939-4010, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  53) Thread[id=4074, name=qtp599160364-4074, state=TIMED_WAITING, group=TGRP-TestOnReconnectListe

[...truncated too long message...]

mpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   12) Thread[id=4048, name=org.eclipse.jetty.server.session.HashSessionManager@21c67c61Timer, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   13) Thread[id=4025, name=OverseerHdfsCoreFailoverThread-96189400141201412-127.0.0.1:62786_-n_0000000000, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
   [junit4]    >         at java.lang.Thread.interrupt0(Native Method)
   [junit4]    >         at java.lang.Thread.interrupt(Thread.java:923)
   [junit4]    >         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   14) Thread[id=4075, name=Scheduler-1455710535, state=TIMED_WAITING, group=TGRP-TestOnReconnectListenerSupport]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   15) Thread[id=4032, name=searcherExecutor-1740-thread-1, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   16) Thread[id=4067, name=qtp599160364-4067, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   17) Thread[id=4051, name=Scheduler-948630609, state=WAITING, group=TGRP-TestOnReconnectListenerSupport]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   18) Thread[id=4059, name=qtp1133684582-4059, state=RUNNABLE, group=TGRP-TestOnReconnectListenerSupport]
   [junit4]    >         at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]    >         at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8008AAF331A6914F]:0)
   [junit4] Completed [137/619 (1!)] on J1 in 184.32s, 1 test, 3 errors <<< FAILURES!

[...truncated 1524 lines...]
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:740: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:684: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/build.xml:59: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/build.xml:233: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/solr/common-build.xml:530: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:1427: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-6.x-Solaris/lucene/common-build.xml:984: There were test failures: 619 suites (10 ignored), 2595 tests, 2 suite-level errors, 1 error, 1527 ignored (63 assumptions) [seed: 8008AAF331A6914F]

Total time: 47 minutes 15 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any