You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2015/03/12 09:13:52 UTC

[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.8.0) - Build # 2050 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/2050/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.core.TestDynamicLoading.testDynamicLoading

Error Message:
Could not get expected value  'org.apache.solr.core.RuntimeLibReqHandler' for path 'overlay/requestHandler/\/runtime/class' full output: {   "responseHeader":{     "status":0,     "QTime":0},   "overlay":{     "znodeVersion":1,     "runtimeLib":{"colltest":{         "name":"colltest",         "version":1}},     "requestHandler":{"/test1":{         "name":"/test1",         "class":"org.apache.solr.core.BlobStoreTestRequestHandler",         "runtimeLib":true}}}}

Stack Trace:
java.lang.AssertionError: Could not get expected value  'org.apache.solr.core.RuntimeLibReqHandler' for path 'overlay/requestHandler/\/runtime/class' full output: {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "overlay":{
    "znodeVersion":1,
    "runtimeLib":{"colltest":{
        "name":"colltest",
        "version":1}},
    "requestHandler":{"/test1":{
        "name":"/test1",
        "class":"org.apache.solr.core.BlobStoreTestRequestHandler",
        "runtimeLib":true}}}}
	at __randomizedtesting.SeedInfo.seed([D859B06A71A2CAA5:149D3D867F6F05]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:399)
	at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:172)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10103 lines...]
   [junit4] Suite: org.apache.solr.core.TestDynamicLoading
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/init-core-data-001
   [junit4]   2> 4660823 T19521 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 4660824 T19521 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /wg_vfc/od
   [junit4]   2> 4660832 T19521 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4660834 T19522 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 4660938 T19521 oasc.ZkTestServer.run start zk server on port:54208
   [junit4]   2> 4660939 T19521 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4660942 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4660954 T19529 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14aaa244 name:ZooKeeperConnection Watcher:127.0.0.1:54208 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4660955 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4660955 T19521 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4660955 T19521 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 4660972 T19521 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4660974 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4660978 T19532 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c0489 name:ZooKeeperConnection Watcher:127.0.0.1:54208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4660979 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4660980 T19521 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4660981 T19521 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 4660990 T19521 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 4660996 T19521 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 4661001 T19521 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 4661013 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4661014 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 4661028 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 4661030 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 4661044 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4661046 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4661053 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4661054 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 4661061 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4661062 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 4661075 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4661076 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 4661090 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4661091 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 4661097 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4661098 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4661114 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4661115 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4661122 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4661123 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 4661138 T19521 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4661139 T19521 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 4663302 T19521 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1
   [junit4]   2> 4663307 T19521 oejs.Server.doStart jetty-9.2.9.v20150224
   [junit4]   2> 4663312 T19521 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@4cfc8633{/wg_vfc/od,null,AVAILABLE}
   [junit4]   2> 4663341 T19521 oejs.AbstractConnector.doStart Started ServerConnector@52efc1bf{HTTP/1.1}{127.0.0.1:५४२११}
   [junit4]   2> 4663342 T19521 oejs.Server.doStart Started @४६७०३०७ms
   [junit4]   2> 4663342 T19521 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/tempDir-001/control/data, hostContext=/wg_vfc/od, hostPort=54211, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores}
   [junit4]   2> 4663343 T19521 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@4554617c
   [junit4]   2> 4663344 T19521 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/'
   [junit4]   2> 4663413 T19521 oasc.SolrXmlConfig.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/solr.xml
   [junit4]   2> 4663428 T19521 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores
   [junit4]   2> 4663428 T19521 oasc.CoreContainer.<init> New CoreContainer 1857702020
   [junit4]   2> 4663429 T19521 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/]
   [junit4]   2> 4663429 T19521 oasc.CoreContainer.load loading shared library: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/lib
   [junit4]   2> 4663430 T19521 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/lib).
   [junit4]   2> 4663444 T19521 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4663445 T19521 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 4663445 T19521 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4663446 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4663446 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 4663446 T19521 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4663447 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4663447 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4663447 T19521 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4663448 T19521 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4663448 T19521 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 4663448 T19521 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 4663449 T19521 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4663449 T19521 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4663450 T19521 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 4663450 T19521 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54208/solr
   [junit4]   2> 4663451 T19521 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 4663451 T19521 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4663453 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4663457 T19546 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48dbb3e5 name:ZooKeeperConnection Watcher:127.0.0.1:54208 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4663457 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4663458 T19521 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4663463 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4663466 T19549 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a0ddd8d name:ZooKeeperConnection Watcher:127.0.0.1:54208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4663467 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4663469 T19521 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 4663477 T19521 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 4663485 T19521 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 4663492 T19521 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 4663499 T19521 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 4663507 T19521 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 4663513 T19521 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 4663518 T19521 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 4663523 T19521 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54211_wg_vfc%2Fod
   [junit4]   2> 4663525 T19521 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54211_wg_vfc%2Fod
   [junit4]   2> 4663530 T19521 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 4663535 T19521 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 4663542 T19521 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 4663546 T19521 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:54211_wg_vfc%2Fod
   [junit4]   2> 4663547 T19521 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 4663551 T19521 oasc.Overseer.start Overseer (id=93463964262334467-127.0.0.1:54211_wg_vfc%2Fod-n_0000000000) starting
   [junit4]   2> 4663557 T19521 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 4663581 T19521 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 4663583 T19551 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 4663583 T19521 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 4663585 T19550 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 4663596 T19521 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 4663599 T19521 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/
   [junit4]   2> 4663599 T19521 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 4663602 T19553 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 4663602 T19553 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 4663604 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4663604 T19553 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 4663607 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54211/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54211_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 4663609 T19550 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54211/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54211_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 4663609 T19550 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 4663609 T19550 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 4663612 T19549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4664606 T19553 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 4664606 T19553 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 4664608 T19553 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 4664609 T19553 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 4664611 T19553 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 4664611 T19553 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/'
   [junit4]   2> 4664636 T19553 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 4664642 T19553 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 4664685 T19553 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 4664690 T19553 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 4664712 T19553 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4664715 T19553 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 4664724 T19553 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 4664879 T19553 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 4664891 T19553 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4664893 T19553 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4664921 T19553 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 4664928 T19553 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 4664938 T19553 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 4664941 T19553 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 4664943 T19553 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 4664943 T19553 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 4664947 T19553 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 4664948 T19553 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 4664948 T19553 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 4664949 T19553 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4664949 T19553 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/data/
   [junit4]   2> 4664949 T19553 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2509e5db
   [junit4]   2> 4664951 T19553 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/data
   [junit4]   2> 4664951 T19553 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/data/index/
   [junit4]   2> 4664952 T19553 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4664953 T19553 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/control-001/cores/collection1/data/index
   [junit4]   2> 4664955 T19553 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.42557307838941294]
   [junit4]   2> 4664956 T19553 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@25010283 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5af7e4ee),segFN=segments_1,generation=1}
   [junit4]   2> 4664956 T19553 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4664964 T19553 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 4664965 T19553 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 4664966 T19553 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 4664967 T19553 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4664967 T19553 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4664968 T19553 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 4664970 T19553 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4664970 T19553 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4664970 T19553 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4664979 T19553 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4664981 T19553 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4664982 T19553 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4664985 T19553 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4664991 T19553 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
   [junit4]   2> 4664992 T19553 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 4664994 T19553 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 4664994 T19553 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 4664996 T19553 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4664996 T19553 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4664996 T19553 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7337882216984942]
   [junit4]   2> 4664997 T19553 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@25010283 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5af7e4ee),segFN=segments_1,generation=1}
   [junit4]   2> 4664997 T19553 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4664998 T19553 oass.SolrIndexSearcher.<init> Opening Searcher@2758569e[collection1] main
   [junit4]   2> 4664998 T19553 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 4665000 T19553 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 4665001 T19553 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 4665002 T19553 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4665002 T19553 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 4665003 T19553 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4665005 T19553 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 4665005 T19553 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4665006 T19553 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 4665006 T19553 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 4665007 T19553 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4665007 T19554 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2758569e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4665008 T19557 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54211/wg_vfc/od collection:control_collection shard:shard1
   [junit4]   2> 4665009 T19521 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 4665010 T19521 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 4665010 T19521 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4665011 T19557 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 4665012 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4665016 T19560 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e1a80f8 name:ZooKeeperConnection Watcher:127.0.0.1:54208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4665017 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4665018 T19521 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4665018 T19521 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 4665025 T19557 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 4665027 T19521 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 4665028 T19521 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 4665028 T19521 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4665029 T19557 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 4665029 T19557 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C6620 name=collection1 org.apache.solr.core.SolrCore@256e45eb url=http://127.0.0.1:54211/wg_vfc/od/collection1 node=127.0.0.1:54211_wg_vfc%2Fod C6620_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:54211/wg_vfc/od, node_name=127.0.0.1:54211_wg_vfc%2Fod, state=down}
   [junit4]   2> 4665030 T19557 C6620 P54211 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54211/wg_vfc/od/collection1/
   [junit4]   2> 4665030 T19557 C6620 P54211 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 4665029 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4665031 T19557 C6620 P54211 oasc.SyncStrategy.syncToMe http://127.0.0.1:54211/wg_vfc/od/collection1/ has no replicas
   [junit4]   2> 4665033 T19557 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54211/wg_vfc/od/collection1/ shard1
   [junit4]   2> 4665034 T19557 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 4665032 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4665036 T19563 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@669f1621 name:ZooKeeperConnection Watcher:127.0.0.1:54208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4665036 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 4665037 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4665038 T19521 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4665044 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 4665044 T19550 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 4665047 T19560 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4665055 T19549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4665056 T19564 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4665063 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54211/wg_vfc/od",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 4665065 T19550 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 4665068 T19560 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4665070 T19549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4665091 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4665195 T19560 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4665196 T19549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4665207 T19557 oasc.ZkController.register We are http://127.0.0.1:54211/wg_vfc/od/collection1/ and leader is http://127.0.0.1:54211/wg_vfc/od/collection1/
   [junit4]   2> 4665208 T19557 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54211/wg_vfc/od
   [junit4]   2> 4665208 T19557 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 4665208 T19557 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 4665208 T19557 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 4665210 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4665215 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54211/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54211_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 4665217 T19550 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54211/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54211_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 4665325 T19549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4665325 T19560 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 4666974 T19521 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1
   [junit4]   2> 4666978 T19521 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001
   [junit4]   2> 4666981 T19521 oejs.Server.doStart jetty-9.2.9.v20150224
   [junit4]   2> 4666988 T19521 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@41c498b0{/wg_vfc/od,null,AVAILABLE}
   [junit4]   2> 4666989 T19521 oejs.AbstractConnector.doStart Started ServerConnector@6b2ed023{HTTP/1.1}{127.0.0.1:५४२१६}
   [junit4]   2> 4666989 T19521 oejs.Server.doStart Started @४६७३९५६ms
   [junit4]   2> 4666990 T19521 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/wg_vfc/od, hostPort=54216, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores}
   [junit4]   2> 4666990 T19521 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@4554617c
   [junit4]   2> 4666992 T19521 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/'
   [junit4]   2> 4667043 T19521 oasc.SolrXmlConfig.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/solr.xml
   [junit4]   2> 4667055 T19521 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores
   [junit4]   2> 4667055 T19521 oasc.CoreContainer.<init> New CoreContainer 2119985935
   [junit4]   2> 4667055 T19521 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/]
   [junit4]   2> 4667056 T19521 oasc.CoreContainer.load loading shared library: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/lib
   [junit4]   2> 4667057 T19521 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/lib).
   [junit4]   2> 4667071 T19521 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4667071 T19521 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 4667072 T19521 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4667072 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4667073 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 4667073 T19521 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4667074 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4667074 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4667075 T19521 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4667075 T19521 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4667075 T19521 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 4667076 T19521 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 4667077 T19521 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4667077 T19521 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4667078 T19521 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 4667079 T19521 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54208/solr
   [junit4]   2> 4667079 T19521 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 4667079 T19521 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4667081 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4667085 T19578 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@eb78625 name:ZooKeeperConnection Watcher:127.0.0.1:54208 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4667086 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4667087 T19521 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4667092 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4667095 T19581 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74e49c8a name:ZooKeeperConnection Watcher:127.0.0.1:54208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4667096 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4667112 T19521 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 4668136 T19521 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54216_wg_vfc%2Fod
   [junit4]   2> 4668139 T19521 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54216_wg_vfc%2Fod
   [junit4]   2> 4668149 T19521 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 4668158 T19521 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1, collection=collection1, absoluteInstDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 4668161 T19521 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/
   [junit4]   2> 4668161 T19521 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 4668170 T19582 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 4668171 T19582 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 4668177 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4668180 T19582 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 4668181 T19582 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 4668180 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54216/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54216_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 4668182 T19582 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 4668183 T19550 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54216/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54216_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 4668183 T19550 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 4668183 T19550 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 4668186 T19582 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 4668186 T19582 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 4668187 T19550 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 4668188 T19581 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 4668190 T19581 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 4669188 T19582 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 4669188 T19582 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 4669192 T19582 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 4669192 T19582 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 4669196 T19582 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 4669196 T19582 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/'
   [junit4]   2> 4669228 T19582 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 4669235 T19582 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 4669252 T19582 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 4669258 T19582 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 4669269 T19582 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4669274 T19582 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 4669280 T19582 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 4669435 T19582 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 4669450 T19582 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4669453 T19582 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4669487 T19582 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 4669494 T19582 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 4669502 T19582 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 4669505 T19582 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 4669506 T19582 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 4669506 T19582 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 4669510 T19582 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 4669511 T19582 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 4669511 T19582 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 4669512 T19582 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4669513 T19582 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 4669514 T19582 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2509e5db
   [junit4]   2> 4669515 T19582 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/data
   [junit4]   2> 4669516 T19582 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 4669516 T19582 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4669517 T19582 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 4669517 T19582 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.42557307838941294]
   [junit4]   2> 4669518 T19582 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@52be5620 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@51febfa8),segFN=segments_1,generation=1}
   [junit4]   2> 4669519 T19582 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4669531 T19582 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 4669532 T19582 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 4669533 T19582 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 4669533 T19582 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4669533 T19582 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4669534 T19582 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 4669534 T19582 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4669534 T19582 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4669535 T19582 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4669543 T19582 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4669546 T19582 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4669548 T19582 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4669550 T19582 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4669556 T19582 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,standard,/update/csv,/update/json/docs,/admin/luke,/admin/segments,/get,/admin/system,/replication,/admin/properties,/config,/schema,/admin/plugins,/admin/logging,/update/json,/admin/threads,/admin/ping,/update,/admin/file
   [junit4]   2> 4669557 T19582 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 4669558 T19582 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 4669558 T19582 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 4669560 T19582 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4669560 T19582 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4669560 T19582 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7337882216984942]
   [junit4]   2> 4669561 T19582 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@52be5620 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@51febfa8),segFN=segments_1,generation=1}
   [junit4]   2> 4669561 T19582 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4669562 T19582 oass.SolrIndexSearcher.<init> Opening Searcher@5235ec82[collection1] main
   [junit4]   2> 4669562 T19582 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 4669565 T19582 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 4669566 T19582 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 4669569 T19582 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4669569 T19582 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 4669570 T19582 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4669572 T19582 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 4669572 T19582 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4669572 T19582 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 4669573 T19582 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 4669575 T19583 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5235ec82[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 4669575 T19582 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4669577 T19586 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54216/wg_vfc/od collection:collection1 shard:shard2
   [junit4]   2> 4669578 T19521 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 4669579 T19521 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 4669580 T19586 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 4669595 T19586 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 4669599 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4669601 T19586 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 4669601 T19586 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C6621 name=collection1 org.apache.solr.core.SolrCore@33af84a0 url=http://127.0.0.1:54216/wg_vfc/od/collection1 node=127.0.0.1:54216_wg_vfc%2Fod C6621_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:54216/wg_vfc/od, node_name=127.0.0.1:54216_wg_vfc%2Fod, state=down}
   [junit4]   2> 4669601 T19586 C6621 P54216 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54216/wg_vfc/od/collection1/
   [junit4]   2> 4669602 T19586 C6621 P54216 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 4669603 T19586 C6621 P54216 oasc.SyncStrategy.syncToMe http://127.0.0.1:54216/wg_vfc/od/collection1/ has no replicas
   [junit4]   2> 4669603 T19586 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54216/wg_vfc/od/collection1/ shard2
   [junit4]   2> 4669604 T19586 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 4669620 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 4669623 T19550 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 4669633 T19581 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 4669636 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54216/wg_vfc/od",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 4669640 T19550 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 4669644 T19581 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 4669647 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4669655 T19587 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 4669660 T19587 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 4669671 T19586 oasc.ZkController.register We are http://127.0.0.1:54216/wg_vfc/od/collection1/ and leader is http://127.0.0.1:54216/wg_vfc/od/collection1/
   [junit4]   2> 4669671 T19586 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54216/wg_vfc/od
   [junit4]   2> 4669672 T19586 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 4669672 T19586 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 4669672 T19586 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 4669676 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4669678 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54216/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54216_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 4669680 T19550 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54216/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54216_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 4669682 T19550 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 4669683 T19587 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 4669694 T19586 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 4669696 T19587 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 4671589 T19521 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1
   [junit4]   2> 4671593 T19521 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001
   [junit4]   2> 4671596 T19521 oejs.Server.doStart jetty-9.2.9.v20150224
   [junit4]   2> 4671601 T19521 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@637ab58e{/wg_vfc/od,null,AVAILABLE}
   [junit4]   2> 4671603 T19521 oejs.AbstractConnector.doStart Started ServerConnector@6f71b4f7{HTTP/1.1}{127.0.0.1:५४२१९}
   [junit4]   2> 4671604 T19521 oejs.Server.doStart Started @४६७८५७३ms
   [junit4]   2> 4671604 T19521 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/wg_vfc/od, hostPort=54219, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores}
   [junit4]   2> 4671605 T19521 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@4554617c
   [junit4]   2> 4671605 T19521 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/'
   [junit4]   2> 4671652 T19521 oasc.SolrXmlConfig.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/solr.xml
   [junit4]   2> 4671664 T19521 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores
   [junit4]   2> 4671664 T19521 oasc.CoreContainer.<init> New CoreContainer 2103458135
   [junit4]   2> 4671665 T19521 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/]
   [junit4]   2> 4671665 T19521 oasc.CoreContainer.load loading shared library: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/lib
   [junit4]   2> 4671666 T19521 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/lib).
   [junit4]   2> 4671684 T19521 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4671685 T19521 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 4671685 T19521 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4671686 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4671686 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 4671687 T19521 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4671687 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4671688 T19521 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4671689 T19521 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4671689 T19521 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4671689 T19521 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 4671690 T19521 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 4671691 T19521 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4671691 T19521 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4671691 T19521 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 4671692 T19521 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54208/solr
   [junit4]   2> 4671693 T19521 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 4671693 T19521 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4671694 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4671700 T19601 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b22b846 name:ZooKeeperConnection Watcher:127.0.0.1:54208 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4671701 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4671702 T19521 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4671708 T19521 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4671713 T19604 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d53fc9b name:ZooKeeperConnection Watcher:127.0.0.1:54208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4671713 T19521 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4671732 T19521 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 4672750 T19521 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54219_wg_vfc%2Fod
   [junit4]   2> 4672752 T19521 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54219_wg_vfc%2Fod
   [junit4]   2> 4672765 T19521 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 4672774 T19521 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores
   [junit4]   1> CORE DESCRIPTOR: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1, collection=collection1, absoluteInstDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 4672780 T19521 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/
   [junit4]   2> 4672782 T19521 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 4672791 T19605 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 4672792 T19605 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 4672796 T19549 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 4672800 T19605 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 4672801 T19605 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 4672800 T19550 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54219/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54219_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 4672803 T19605 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 4672803 T19550 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54219/wg_vfc/od",
   [junit4]   2> 	  "node_name":"127.0.0.1:54219_wg_vfc%2Fod",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 4672805 T19550 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 4672805 T19550 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 4672807 T19605 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 4672808 T19550 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 4672808 T19605 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 4672810 T19604 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 4672811 T19587 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 4672813 T19604 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 4672818 T19587 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 4673810 T19605 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 4673811 T19605 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 4673812 T19605 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 4673813 T19605 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 4673816 T19605 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 4673816 T19605 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/'
   [junit4]   2> 4673852 T19605 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 4673857 T19605 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 4673866 T19605 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 4673876 T19605 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 4673886 T19605 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4673897 T19605 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 4673903 T19605 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 4674099 T19605 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 4674115 T19605 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4674118 T19605 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4674143 T19605 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 4674150 T19605 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 4674155 T19605 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 4674161 T19605 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 4674161 T19605 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 4674162 T19605 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 4674164 T19605 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 4674165 T19605 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 4674165 T19605 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 4674165 T19605 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4674166 T19605 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/data/
   [junit4]   2> 4674166 T19605 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2509e5db
   [junit4]   2> 4674166 T19605 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/data
   [junit4]   2> 4674167 T19605 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/data/index/
   [junit4]   2> 4674167 T19605 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4674168 T19605 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.core.TestDynamicLoading D859B06A71A2CAA5-001/shard-2-001/cores/collection1/data/index
   [junit4]   2> 4674168 T19605 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.42557307838941294]
   [junit4]   2> 4674169 T19605 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4af0b7a9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55069d78),segFN=segments_1,generation=1}
   [junit4]   2> 4674169 T19605 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4674179 T19605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 4674180 T19605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 4674180 T19605 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 4674180 T19605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4674181 T19605 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4674181 T19605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 4674181 T19605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcess

[...truncated too long message...]

sOrgProviderTest, PreAnalyzedUpdateProcessorTest, PrimitiveFieldTypeTest, TestWriterPerf, BasicDistributedZk2Test, SolrInfoMBeanTest, TestSchemaNameResource, SynonymTokenizerTest, TestBinaryResponseWriter, DisMaxRequestHandlerTest, TestAnalyzeInfixSuggestions, SimpleMLTQParserTest, TestFieldSortValues, TestIndexingPerformance, TestExtendedDismaxParser, LeaderElectionIntegrationTest, BJQParserTest, ShardRoutingCustomTest, TestQuerySenderListener, FieldAnalysisRequestHandlerTest, SolrRequestParserTest, TestSolrQueryParser, TestManagedStopFilterFactory, NoCacheHeaderTest, TestTrackingShardHandlerFactory, ActionThrottleTest, SchemaVersionSpecificBehaviorTest, TestRecoveryHdfs, UpdateRequestProcessorFactoryTest, TestSchemaResource, AtomicUpdatesTest, OverseerStatusTest, ResourceLoaderTest, TestManagedResource, TestFastWriter, TestCustomSort, DeleteInactiveReplicaTest, TestCloudSchemaless, MultiThreadedOCPTest, HighlighterTest, TestStandardQParsers, ClusterStateTest, TestHashPartitioner, ExternalFileFieldSortTest, PluginInfoTest, TestExceedMaxTermLength, TestFieldResource, CurrencyFieldXmlFileTest, TestWordDelimiterFilterFactory, TestCollapseQParserPlugin, StatelessScriptUpdateProcessorFactoryTest, TestCopyFieldCollectionResource, TestCryptoKeys, TestSolrXml, TestDFRSimilarityFactory, CachingDirectoryFactoryTest, BasicFunctionalityTest, TestDefaultSimilarityFactory, OutputWriterTest, TestFreeTextSuggestions, TestObjectReleaseTracker, TestSolrQueryParserResource, TestJmxMonitoredMap, TestSchemaManager, TestReqParamsAPI, SpellCheckCollatorTest, SpellCheckComponentTest, TestXIncludeConfig, TestJoin, FullSolrCloudDistribCmdsTest, TestNonDefinedSimilarityFactory, DirectUpdateHandlerOptimizeTest, ZkCLITest, TestHashQParserPlugin, TestStressUserVersions, TestSolrDeletionPolicy1, TestSolrDeletionPolicy2, TestQuerySenderNoQuery, HdfsUnloadDistributedZkTest, SpatialFilterTest, TermsComponentTest, TestZkChroot, TestRecovery, TriLevelCompositeIdRoutingTest, TestNRTOpen, CoreAdminRequestStatusTest, StandardRequestHandlerTest, HighlighterMaxOffsetTest, TestExactStatsCache, TestSolrIndexConfig, ClusterStateUpdateTest, TestDistribDocBasedVersion, TestFaceting, TestCharFilters, ResponseHeaderTest, TestFunctionQuery, TestBulkSchemaAPI, TestCloudInspectUtil, EchoParamsTest, SuggestComponentTest, TestConfig, TestPartialUpdateDeduplication, AliasIntegrationTest, AutoCommitTest, TestShardHandlerFactory, DistribCursorPagingTest, CoreAdminHandlerTest, SyncSliceTest, QueryEqualityTest, SegmentsInfoRequestHandlerTest, TestReplicationHandlerBackup, HdfsDirectoryFactoryTest, TestFieldCollectionResource, LukeRequestHandlerTest, TestDistributedMissingSort, ConnectionManagerTest, TestSearchPerf, IndexSchemaRuntimeFieldTest, ShowFileRequestHandlerTest, CloudMLTQParserTest, TestSearcherReuse, MinimalSchemaTest, TestDistributedSearch, FieldMutatingUpdateProcessorTest, OverseerRolesTest, DirectSolrSpellCheckerTest, SolrIndexSplitterTest, OverriddenZkACLAndCredentialsProvidersTest, DeleteShardTest, TestIntervalFaceting, ScriptEngineTest, TestFastOutputStream, TestAnalyzedSuggestions, TestBlendedInfixSuggestions, TestSolrConfigHandler, SSLMigrationTest, TestSolrQueryParserDefaultOperatorResource, SpellingQueryConverterTest, TestSuggestSpellingConverter, AsyncMigrateRouteKeyTest, DistribJoinFromCollectionTest, LoggingHandlerTest, OverseerCollectionProcessorTest, DistributedMLTComponentTest, MBeansHandlerTest, TestImplicitCoreProperties, TestIndexSearcher, ShardSplitTest, TestCollationField, ExitableDirectoryReaderTest, TestDocumentBuilder, TestCoreContainer, TestJmxIntegration, TestOverriddenPrefixQueryForCustomFieldType, ReplicationFactorTest, AnalysisErrorHandlingTest, HighlighterConfigTest, TestManagedSchemaFieldTypeResource, ResponseLogComponentTest, DistributedDebugComponentTest, DefaultValueUpdateProcessorTest, TestFiltering, OutOfBoxZkACLAndCredentialsProvidersTest, TestDocSet, MoreLikeThisHandlerTest, TestSolrDynamicMBean, TestRangeQuery, HdfsDirectoryTest, TestSolr4Spatial2, IndexBasedSpellCheckerTest, MultiTermTest, DirectUpdateHandlerTest, TestSchemaVersionResource, DistributedQueryComponentCustomSortTest, TestAtomicUpdateErrorCases, CacheHeaderTest, TestDistributedGrouping, TestCloudPivotFacet, PolyFieldTest, SharedFSAutoReplicaFailoverUtilsTest, MergeStrategyTest, BufferStoreTest, TestQueryUtils, UpdateParamsTest, HdfsWriteToMultipleCollectionsTest, TestPostingsSolrHighlighter, SharedFSAutoReplicaFailoverTest, TestBinaryField, TestSimpleTrackingShardHandler, UnloadDistributedZkTest, TestUtils, TimeZoneUtilsTest, DistributedSuggestComponentTest, XmlUpdateRequestHandlerTest, FacetPivotSmallTest, DistributedFacetPivotSmallTest, TestRandomMergePolicy, PreAnalyzedFieldTest, BasicDistributedZkTest, TestLFUCache, TestReRankQParserPlugin, SimpleCollectionCreateDeleteTest, AnalyticsQueryTest, TestValueSourceCache, TestCoreDiscovery, DistributedFacetPivotLargeTest, PeerSyncTest, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, ShardRoutingTest, BasicZkTest, RecoveryZkTest, TestReplicationHandler, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, TestRandomDVFaceting, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, ZkControllerTest, TestRealTimeGet, TestStressReorder, TestReload, TestStressVersions, HardAutoCommitTest, DistributedTermsComponentTest, SimpleFacetsTest, SolrCoreTest, TestGroupingSearch, SolrCmdDistributorTest, QueryElevationComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestSort, TestLazyCores, TestBadConfig, SoftAutoCommitTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, TestFoldingMultitermQuery, SuggesterTSTTest, SuggesterTest, TestStressLucene, TestTrie, SuggesterWFSTTest, TestCSVLoader, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestUpdate, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, XsltUpdateRequestHandlerTest, DebugComponentTest, TestSurroundQueryParser, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, TestQueryTypes, FileBasedSpellCheckerTest, RequestHandlersTest, DocumentBuilderTest, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, TestArbitraryIndexDir, FastVectorHighlighterTest, RegexBoostProcessorTest, SolrPluginUtilsTest, IndexSchemaTest, ReturnFieldsTest, JSONWriterTest, QueryParsingTest, JsonLoaderTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, PingRequestHandlerTest, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, SOLR749Test, AlternateDirectoryTest, CopyFieldTest, BadComponentTest, TestStressRecovery, TestMergePolicyConfig, SampleTest, NumericFieldsTest, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestPhraseSuggestions, TestCodecSupport, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, URLClassifyProcessorTest, TestFastLRUCache, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, TestSystemIdResolver, PrimUtilsTest, DateFieldTest, DOMUtilTest, RAMDirectoryFactoryTest, TestSolrJ, ZkNodePropsTest, SystemInfoHandlerTest, UUIDFieldTest, FileUtilsTest, CircularListTest, TestRTGBase, CursorPagingTest, DistributedIntervalFacetingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestTolerantSearch, AssignTest, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, CustomCollectionTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, DistributedQueueTest, ExternalCollectionsTest, HttpPartitionTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnCommitTest, MigrateRouteKeyTest, RecoveryAfterSoftCommitTest, RemoteQueryErrorTest, RollingRestartTest, SolrXmlInZkTest, TestCollectionAPI, TestLeaderElectionZkExpiry, TestMiniSolrCloudCluster, TestMiniSolrCloudClusterSSL, TestRebalanceLeaders, TestReplicaProperties, TestRequestStatusCollectionAPI, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZk2Test, HdfsBasicDistributedZkTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, HdfsRecoveryZkTest, StressHdfsTest, TestClusterStateMutator, ZkStateWriterTest, QueryResultKeyTest, TestConfigOverlay, TestConfigSets, TestDynamicLoading]
   [junit4] Completed in 59.60s, 1 test, 1 failure <<< FAILURES!

[...truncated 232 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:519: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:467: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:61: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/extra-targets.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:191: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:510: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1351: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:958: There were test failures: 473 suites, 1882 tests, 1 failure, 53 ignored (20 assumptions)

Total time: 131 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.8.0) - Build # 2051 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/2051/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

4 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Didn't see replicas [core_node2, core_node3] come up within 90000 ms! ClusterState: DocCollection(c8n_1x3_lf)={   "replicationFactor":"3",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node1":{           "core":"c8n_1x3_lf_shard1_replica2",           "base_url":"http://127.0.0.1:50907/_yrq/b",           "node_name":"127.0.0.1:50907__yrq%2Fb",           "state":"down"},         "core_node2":{           "core":"c8n_1x3_lf_shard1_replica1",           "base_url":"http://127.0.0.1:50902/_yrq/b",           "node_name":"127.0.0.1:50902__yrq%2Fb",           "state":"recovering"},         "core_node3":{           "core":"c8n_1x3_lf_shard1_replica3",           "base_url":"http://127.0.0.1:50911/_yrq/b",           "node_name":"127.0.0.1:50911__yrq%2Fb",           "state":"active",           "leader":"true"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Didn't see replicas [core_node2, core_node3] come up within 90000 ms! ClusterState: DocCollection(c8n_1x3_lf)={
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "core":"c8n_1x3_lf_shard1_replica2",
          "base_url":"http://127.0.0.1:50907/_yrq/b",
          "node_name":"127.0.0.1:50907__yrq%2Fb",
          "state":"down"},
        "core_node2":{
          "core":"c8n_1x3_lf_shard1_replica1",
          "base_url":"http://127.0.0.1:50902/_yrq/b",
          "node_name":"127.0.0.1:50902__yrq%2Fb",
          "state":"recovering"},
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica3",
          "base_url":"http://127.0.0.1:50911/_yrq/b",
          "node_name":"127.0.0.1:50911__yrq%2Fb",
          "state":"active",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false"}
	at __randomizedtesting.SeedInfo.seed([8A9D0DE972E4F27F:2C93233DC189F87]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.HttpPartitionTest.waitToSeeReplicasActive(HttpPartitionTest.java:572)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:178)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.util.TestUtils.testNanoTimeSpeed

Error Message:
Time taken for System.nanoTime is too high

Stack Trace:
java.lang.AssertionError: Time taken for System.nanoTime is too high
	at __randomizedtesting.SeedInfo.seed([8A9D0DE972E4F27F:FC6719DF7E4F292F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.util.TestUtils.testNanoTimeSpeed(TestUtils.java:197)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.util.TestUtils

Error Message:
17 threads leaked from SUITE scope at org.apache.solr.util.TestUtils:     1) Thread[id=17098, name=nanoTimeTestThread-8030-thread-3, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=17111, name=nanoTimeTestThread-8030-thread-16, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=17097, name=nanoTimeTestThread-8030-thread-2, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=17099, name=nanoTimeTestThread-8030-thread-4, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=17104, name=nanoTimeTestThread-8030-thread-9, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=17112, name=nanoTimeTestThread-8030-thread-17, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=17102, name=nanoTimeTestThread-8030-thread-7, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=17109, name=nanoTimeTestThread-8030-thread-14, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=17106, name=nanoTimeTestThread-8030-thread-11, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=17100, name=nanoTimeTestThread-8030-thread-5, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=17096, name=nanoTimeTestThread-8030-thread-1, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=17105, name=nanoTimeTestThread-8030-thread-10, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=17101, name=nanoTimeTestThread-8030-thread-6, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=17107, name=nanoTimeTestThread-8030-thread-12, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=17103, name=nanoTimeTestThread-8030-thread-8, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=17108, name=nanoTimeTestThread-8030-thread-13, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   17) Thread[id=17110, name=nanoTimeTestThread-8030-thread-15, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 17 threads leaked from SUITE scope at org.apache.solr.util.TestUtils: 
   1) Thread[id=17098, name=nanoTimeTestThread-8030-thread-3, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=17111, name=nanoTimeTestThread-8030-thread-16, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=17097, name=nanoTimeTestThread-8030-thread-2, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=17099, name=nanoTimeTestThread-8030-thread-4, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=17104, name=nanoTimeTestThread-8030-thread-9, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=17112, name=nanoTimeTestThread-8030-thread-17, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=17102, name=nanoTimeTestThread-8030-thread-7, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=17109, name=nanoTimeTestThread-8030-thread-14, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=17106, name=nanoTimeTestThread-8030-thread-11, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=17100, name=nanoTimeTestThread-8030-thread-5, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=17096, name=nanoTimeTestThread-8030-thread-1, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=17105, name=nanoTimeTestThread-8030-thread-10, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=17101, name=nanoTimeTestThread-8030-thread-6, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=17107, name=nanoTimeTestThread-8030-thread-12, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=17103, name=nanoTimeTestThread-8030-thread-8, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=17108, name=nanoTimeTestThread-8030-thread-13, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  17) Thread[id=17110, name=nanoTimeTestThread-8030-thread-15, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([8A9D0DE972E4F27F]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.util.TestUtils

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=17098, name=nanoTimeTestThread-8030-thread-3, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=17111, name=nanoTimeTestThread-8030-thread-16, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=17097, name=nanoTimeTestThread-8030-thread-2, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=17099, name=nanoTimeTestThread-8030-thread-4, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=17104, name=nanoTimeTestThread-8030-thread-9, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=17112, name=nanoTimeTestThread-8030-thread-17, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=17102, name=nanoTimeTestThread-8030-thread-7, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=17109, name=nanoTimeTestThread-8030-thread-14, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=17106, name=nanoTimeTestThread-8030-thread-11, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=17100, name=nanoTimeTestThread-8030-thread-5, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=17096, name=nanoTimeTestThread-8030-thread-1, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=17105, name=nanoTimeTestThread-8030-thread-10, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=17101, name=nanoTimeTestThread-8030-thread-6, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=17107, name=nanoTimeTestThread-8030-thread-12, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=17103, name=nanoTimeTestThread-8030-thread-8, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=17108, name=nanoTimeTestThread-8030-thread-13, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   17) Thread[id=17110, name=nanoTimeTestThread-8030-thread-15, state=TIMED_WAITING, group=TGRP-TestUtils]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=17098, name=nanoTimeTestThread-8030-thread-3, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=17111, name=nanoTimeTestThread-8030-thread-16, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=17097, name=nanoTimeTestThread-8030-thread-2, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=17099, name=nanoTimeTestThread-8030-thread-4, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=17104, name=nanoTimeTestThread-8030-thread-9, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=17112, name=nanoTimeTestThread-8030-thread-17, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=17102, name=nanoTimeTestThread-8030-thread-7, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=17109, name=nanoTimeTestThread-8030-thread-14, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=17106, name=nanoTimeTestThread-8030-thread-11, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=17100, name=nanoTimeTestThread-8030-thread-5, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=17096, name=nanoTimeTestThread-8030-thread-1, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=17105, name=nanoTimeTestThread-8030-thread-10, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=17101, name=nanoTimeTestThread-8030-thread-6, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=17107, name=nanoTimeTestThread-8030-thread-12, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=17103, name=nanoTimeTestThread-8030-thread-8, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=17108, name=nanoTimeTestThread-8030-thread-13, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  17) Thread[id=17110, name=nanoTimeTestThread-8030-thread-15, state=TIMED_WAITING, group=TGRP-TestUtils]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([8A9D0DE972E4F27F]:0)




Build Log:
[...truncated 9195 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/init-core-data-001
   [junit4]   2> 1576260 T5354 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_yrq/b
   [junit4]   2> 1576267 T5354 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1576270 T5355 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1576370 T5354 oasc.ZkTestServer.run start zk server on port:50899
   [junit4]   2> 1576370 T5354 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1576372 T5354 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1576379 T5362 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@baf515c name:ZooKeeperConnection Watcher:127.0.0.1:50899 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1576380 T5354 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1576380 T5354 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1576380 T5354 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1576410 T5354 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1576415 T5354 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1576424 T5365 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4392946a name:ZooKeeperConnection Watcher:127.0.0.1:50899/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1576424 T5354 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1576424 T5354 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1576424 T5354 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1576428 T5354 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1576433 T5354 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1576440 T5354 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1576446 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1576446 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1576451 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1576451 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1576457 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1576458 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1576463 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1576463 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1576469 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1576469 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1576474 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1576474 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1576479 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1576480 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1576484 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1576485 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1576490 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1576490 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1576495 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1576495 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1576499 T5354 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1576500 T5354 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1578157 T5354 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores/collection1
   [junit4]   2> 1578162 T5354 oejs.Server.doStart jetty-9.2.9.v20150224
   [junit4]   2> 1578167 T5354 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@1349635c{/_yrq/b,null,AVAILABLE}
   [junit4]   2> 1578170 T5354 oejs.AbstractConnector.doStart Started ServerConnector@7a9b8baf{HTTP/1.1}{127.0.0.1:50903}
   [junit4]   2> 1578170 T5354 oejs.Server.doStart Started @1582793ms
   [junit4]   2> 1578171 T5354 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/tempDir-001/control/data, hostContext=/_yrq/b, hostPort=50902, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores}
   [junit4]   2> 1578171 T5354 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@4554617c
   [junit4]   2> 1578172 T5354 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/'
   [junit4]   2> 1578229 T5354 oasc.SolrXmlConfig.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/solr.xml
   [junit4]   2> 1578257 T5354 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores
   [junit4]   2> 1578258 T5354 oasc.CoreContainer.<init> New CoreContainer 2083940178
   [junit4]   2> 1578258 T5354 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/]
   [junit4]   2> 1578258 T5354 oasc.CoreContainer.load loading shared library: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/lib
   [junit4]   2> 1578259 T5354 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/lib).
   [junit4]   2> 1578270 T5354 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1578270 T5354 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1578270 T5354 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1578271 T5354 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1578271 T5354 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1578272 T5354 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1578272 T5354 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1578272 T5354 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1578272 T5354 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1578273 T5354 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1578273 T5354 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1578273 T5354 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1578274 T5354 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1578274 T5354 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1578274 T5354 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1578275 T5354 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50899/solr
   [junit4]   2> 1578275 T5354 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1578276 T5354 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1578277 T5354 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1578281 T5379 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29ed68ec name:ZooKeeperConnection Watcher:127.0.0.1:50899 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1578282 T5354 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1578283 T5354 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1578294 T5354 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1578295 T5382 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@79289c75 name:ZooKeeperConnection Watcher:127.0.0.1:50899/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1578296 T5354 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1578299 T5354 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1578310 T5354 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1578317 T5354 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1578324 T5354 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1578332 T5354 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1578340 T5354 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1578350 T5354 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1578354 T5354 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1578358 T5354 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50902__yrq%2Fb
   [junit4]   2> 1578360 T5354 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50902__yrq%2Fb
   [junit4]   2> 1578366 T5354 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1578372 T5354 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1578383 T5354 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1578388 T5354 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:50902__yrq%2Fb
   [junit4]   2> 1578388 T5354 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1578396 T5354 oasc.Overseer.start Overseer (id=93466154206035971-127.0.0.1:50902__yrq%2Fb-n_0000000000) starting
   [junit4]   2> 1578403 T5354 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1578425 T5354 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1578426 T5384 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1578427 T5354 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1578427 T5383 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1578439 T5354 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, instanceDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores/collection1/, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 1578442 T5354 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores/collection1/
   [junit4]   2> 1578442 T5354 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1578450 T5386 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1578451 T5386 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1578456 T5386 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1578473 T5383 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50902/_yrq/b",
   [junit4]   2> 	  "node_name":"127.0.0.1:50902__yrq%2Fb",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1578477 T5383 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50902/_yrq/b",
   [junit4]   2> 	  "node_name":"127.0.0.1:50902__yrq%2Fb",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1578492 T5383 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1578493 T5383 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1578497 T5382 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1578504 T5382 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1579458 T5386 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1579459 T5386 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1579460 T5386 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1579460 T5386 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1579462 T5386 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1579462 T5386 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores/collection1/'
   [junit4]   2> 1579492 T5386 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1579496 T5386 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1579506 T5386 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1579511 T5386 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 1579534 T5386 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1579537 T5386 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1579541 T5386 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1579736 T5386 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1579745 T5386 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1579747 T5386 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1579768 T5386 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1579774 T5386 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1579778 T5386 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1579780 T5386 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1579781 T5386 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1579781 T5386 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1579783 T5386 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1579783 T5386 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1579784 T5386 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1579784 T5386 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1579784 T5386 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores/collection1/, dataDir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest 8A9D0DE972E4F27F-001/control-001/cores/collection1/data/
   [junit4]   2> 1579784 T5386 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.Jm

[...truncated too long message...]

til.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   10) Thread[id=17100, name=nanoTimeTestThread-8030-thread-5, state=TIMED_WAITING, group=TGRP-TestUtils]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   11) Thread[id=17096, name=nanoTimeTestThread-8030-thread-1, state=TIMED_WAITING, group=TGRP-TestUtils]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   12) Thread[id=17105, name=nanoTimeTestThread-8030-thread-10, state=TIMED_WAITING, group=TGRP-TestUtils]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   13) Thread[id=17101, name=nanoTimeTestThread-8030-thread-6, state=TIMED_WAITING, group=TGRP-TestUtils]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   14) Thread[id=17107, name=nanoTimeTestThread-8030-thread-12, state=TIMED_WAITING, group=TGRP-TestUtils]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   15) Thread[id=17103, name=nanoTimeTestThread-8030-thread-8, state=TIMED_WAITING, group=TGRP-TestUtils]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   16) Thread[id=17108, name=nanoTimeTestThread-8030-thread-13, state=TIMED_WAITING, group=TGRP-TestUtils]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   17) Thread[id=17110, name=nanoTimeTestThread-8030-thread-15, state=TIMED_WAITING, group=TGRP-TestUtils]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8A9D0DE972E4F27F]:0)
   [junit4] Completed in 24.54s, 6 tests, 1 failure, 2 errors <<< FAILURES!

[...truncated 359 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:519: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:467: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:61: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/extra-targets.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:191: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:510: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1351: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:958: There were test failures: 473 suites, 1883 tests, 2 suite-level errors, 2 failures, 350 ignored (20 assumptions)

Total time: 110 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any