You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2015/02/24 10:24:31 UTC

[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.8.0_40-ea-b22) - Build # 11866 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/11866/
Java: 64bit/jdk1.8.0_40-ea-b22 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
commitWithin did not work on node: http://127.0.0.1:46408/v_fry/eq/collection1 expected:<68> but was:<67>

Stack Trace:
java.lang.AssertionError: commitWithin did not work on node: http://127.0.0.1:46408/v_fry/eq/collection1 expected:<68> but was:<67>
	at __randomizedtesting.SeedInfo.seed([CA15EC54DE23A8E3:4241D38E70DFC51B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:343)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9581 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/init-core-data-001
   [junit4]   2> 752595 T4391 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /v_fry/eq
   [junit4]   2> 752599 T4391 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 752600 T4392 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 752699 T4391 oasc.ZkTestServer.run start zk server on port:58208
   [junit4]   2> 752700 T4391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 752716 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 752720 T4399 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@db4c36b name:ZooKeeperConnection Watcher:127.0.0.1:58208 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 752721 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 752721 T4391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 752721 T4391 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 752732 T4391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 752733 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 752734 T4402 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@520f31ad name:ZooKeeperConnection Watcher:127.0.0.1:58208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 752735 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 752735 T4391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 752735 T4391 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 752737 T4391 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 752738 T4391 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 752739 T4391 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 752741 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 752741 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 752743 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 752743 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 752744 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 752745 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 752746 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 752747 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 752749 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 752749 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 752750 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 752751 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 752752 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 752753 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 752757 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 752758 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 752765 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 752766 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 752767 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 752767 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 752768 T4391 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 752769 T4391 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 753108 T4391 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/cores/collection1
   [junit4]   2> 753109 T4391 oejs.Server.doStart jetty-9.2.6.v20141205
   [junit4]   2> 753111 T4391 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@3daf737e{/v_fry/eq,null,AVAILABLE}
   [junit4]   2> 753116 T4391 oejs.AbstractConnector.doStart Started ServerConnector@2688a31d{HTTP/1.1}{127.0.0.1:35314}
   [junit4]   2> 753117 T4391 oejs.Server.doStart Started @754375ms
   [junit4]   2> 753117 T4391 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@4e0e2f2a
   [junit4]   2> 753117 T4391 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 753118 T4391 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001
   [junit4]   2> 753118 T4391 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/'
   [junit4]   2> 753129 T4391 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/solr.xml
   [junit4]   2> 753134 T4391 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/cores
   [junit4]   2> 753134 T4391 oasc.CoreContainer.<init> New CoreContainer 1948598853
   [junit4]   2> 753135 T4391 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/]
   [junit4]   2> 753135 T4391 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 753136 T4391 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 753136 T4391 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 753136 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 753137 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 753137 T4391 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 753137 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 753138 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 753138 T4391 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 753138 T4391 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 753138 T4391 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 753139 T4391 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 753140 T4391 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 753140 T4391 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 753140 T4391 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 753140 T4391 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58208/solr
   [junit4]   2> 753141 T4391 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 753141 T4391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 753143 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 753168 T4416 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1212b7d4 name:ZooKeeperConnection Watcher:127.0.0.1:58208 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 753168 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 753169 T4391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 753170 T4393 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14bbad27af00002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 753171 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 753190 T4419 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7813f11b name:ZooKeeperConnection Watcher:127.0.0.1:58208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 753195 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 753196 T4391 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 753198 T4391 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 753200 T4391 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 753202 T4391 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 753207 T4391 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 753211 T4391 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 753213 T4391 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35314_v_fry%2Feq
   [junit4]   2> 753214 T4391 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:35314_v_fry%2Feq
   [junit4]   2> 753216 T4391 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 753217 T4391 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 753218 T4391 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 753219 T4391 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:35314_v_fry%2Feq
   [junit4]   2> 753220 T4391 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 753221 T4391 oasc.Overseer.start Overseer (id=93373630459674627-127.0.0.1:35314_v_fry%2Feq-n_0000000000) starting
   [junit4]   2> 753222 T4391 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 753226 T4391 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 753227 T4421 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 753228 T4420 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 753228 T4391 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 753230 T4391 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 753231 T4391 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 753236 T4391 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/cores
   [junit4]   2> 753237 T4391 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/cores/collection1/
   [junit4]   2> 753237 T4391 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 753238 T4423 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 753238 T4423 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 753238 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753238 T4423 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 753239 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35314/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:35314_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 753239 T4420 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35314/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:35314_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 753239 T4420 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 753240 T4420 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 753241 T4419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 754239 T4423 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 754239 T4423 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 754240 T4423 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 754240 T4423 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 754240 T4423 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 754241 T4423 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/cores/collection1/'
   [junit4]   2> 754253 T4423 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 754256 T4423 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 754261 T4423 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 754266 T4423 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 754276 T4423 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 754277 T4423 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 754290 T4423 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 754430 T4423 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 754441 T4423 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 754443 T4423 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 754452 T4423 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 754455 T4423 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 754457 T4423 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 754458 T4423 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 754458 T4423 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 754458 T4423 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 754459 T4423 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 754459 T4423 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 754459 T4423 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 754459 T4423 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 754460 T4423 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/control-001/cores/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/control/data/
   [junit4]   2> 754460 T4423 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@157457a3
   [junit4]   2> 754461 T4423 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/control/data
   [junit4]   2> 754461 T4423 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/control/data/index/
   [junit4]   2> 754461 T4423 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 754461 T4423 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/control/data/index
   [junit4]   2> 754462 T4423 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6283384548054446]
   [junit4]   2> 754463 T4423 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@741bfcb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e580569),segFN=segments_1,generation=1}
   [junit4]   2> 754463 T4423 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 754465 T4423 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 754466 T4423 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 754466 T4423 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 754466 T4423 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 754466 T4423 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 754467 T4423 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 754467 T4423 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 754467 T4423 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 754467 T4423 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 754468 T4423 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 754468 T4423 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 754468 T4423 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 754469 T4423 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 754469 T4423 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 754469 T4423 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 754470 T4423 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 754470 T4423 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 754470 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 754471 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 754471 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 754471 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 754472 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 754472 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 754472 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 754473 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 754473 T4423 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 754473 T4423 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 754476 T4423 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 754477 T4423 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 754477 T4423 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 754478 T4423 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 754479 T4423 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 754480 T4423 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 754480 T4423 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 754481 T4423 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=58.2373046875, floorSegmentMB=1.5234375, forceMergeDeletesPctAllowed=9.088801577506036, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24295251126522147
   [junit4]   2> 754482 T4423 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@741bfcb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e580569),segFN=segments_1,generation=1}
   [junit4]   2> 754482 T4423 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 754482 T4423 oass.SolrIndexSearcher.<init> Opening Searcher@4a5d0c26[collection1] main
   [junit4]   2> 754482 T4423 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 754483 T4423 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 754483 T4423 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 754484 T4423 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 754484 T4423 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 754484 T4423 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 754484 T4423 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 754485 T4423 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 754485 T4423 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 754485 T4423 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 754486 T4423 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 754486 T4424 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4a5d0c26[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 754486 T4427 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:35314/v_fry/eq collection:control_collection shard:shard1
   [junit4]   2> 754486 T4391 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 754487 T4391 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 754487 T4427 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 754487 T4391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 754488 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 754491 T4427 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 754492 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 754492 T4427 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 754492 T4427 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2158 name=collection1 org.apache.solr.core.SolrCore@4a04f35c url=http://127.0.0.1:35314/v_fry/eq/collection1 node=127.0.0.1:35314_v_fry%2Feq C2158_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:35314/v_fry/eq, node_name=127.0.0.1:35314_v_fry%2Feq, state=down}
   [junit4]   2> 754492 T4427 C2158 P35314 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:35314/v_fry/eq/collection1/
   [junit4]   2> 754492 T4427 C2158 P35314 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 754493 T4427 C2158 P35314 oasc.SyncStrategy.syncToMe http://127.0.0.1:35314/v_fry/eq/collection1/ has no replicas
   [junit4]   2> 754493 T4427 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:35314/v_fry/eq/collection1/ shard1
   [junit4]   2> 754493 T4427 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 754493 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 754494 T4430 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d786449 name:ZooKeeperConnection Watcher:127.0.0.1:58208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 754495 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 754501 T4391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 754502 T4391 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 754502 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 754503 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35314/v_fry/eq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 754504 T4391 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 754605 T4419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 754605 T4430 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 754653 T4427 oasc.ZkController.register We are http://127.0.0.1:35314/v_fry/eq/collection1/ and leader is http://127.0.0.1:35314/v_fry/eq/collection1/
   [junit4]   2> 754653 T4427 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:35314/v_fry/eq
   [junit4]   2> 754654 T4427 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 754654 T4427 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 754654 T4427 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 754655 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 754656 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35314/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:35314_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 2
   [junit4]   2> 754656 T4420 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35314/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:35314_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 754758 T4430 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 754759 T4419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 754893 T4391 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/cores/collection1
   [junit4]   2> 754894 T4391 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 754899 T4391 oejs.Server.doStart jetty-9.2.6.v20141205
   [junit4]   2> 754901 T4391 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@1d1ef0c3{/v_fry/eq,null,AVAILABLE}
   [junit4]   2> 754901 T4391 oejs.AbstractConnector.doStart Started ServerConnector@318bca82{HTTP/1.1}{127.0.0.1:46408}
   [junit4]   2> 754902 T4391 oejs.Server.doStart Started @756161ms
   [junit4]   2> 754902 T4391 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@4e0e2f2a
   [junit4]   2> 754903 T4391 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 754904 T4391 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001
   [junit4]   2> 754904 T4391 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/'
   [junit4]   2> 754920 T4391 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/solr.xml
   [junit4]   2> 754928 T4391 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/cores
   [junit4]   2> 754929 T4391 oasc.CoreContainer.<init> New CoreContainer 1106376540
   [junit4]   2> 754929 T4391 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/]
   [junit4]   2> 754930 T4391 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 754930 T4391 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 754930 T4391 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 754931 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 754931 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 754931 T4391 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 754932 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 754932 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 754932 T4391 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 754932 T4391 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 754933 T4391 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 754933 T4391 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 754934 T4391 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 754934 T4391 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 754934 T4391 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 754935 T4391 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58208/solr
   [junit4]   2> 754935 T4391 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 754935 T4391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 754940 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 754941 T4444 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@670ab6e5 name:ZooKeeperConnection Watcher:127.0.0.1:58208 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 754941 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 754942 T4391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 754953 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 754963 T4447 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b473096 name:ZooKeeperConnection Watcher:127.0.0.1:58208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 754963 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 754967 T4391 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 755970 T4391 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46408_v_fry%2Feq
   [junit4]   2> 755971 T4391 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:46408_v_fry%2Feq
   [junit4]   2> 755974 T4391 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 755976 T4391 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/cores
   [junit4]   2> 755977 T4391 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/cores/collection1/
   [junit4]   2> 755977 T4391 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 755978 T4448 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 755978 T4448 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 755979 T4448 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 755979 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 755980 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46408/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:46408_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 755981 T4420 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46408/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:46408_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 755981 T4420 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 755981 T4420 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 756083 T4447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756083 T4430 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756083 T4419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756980 T4448 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 756980 T4448 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 756981 T4448 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 756981 T4448 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 756986 T4448 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 756986 T4448 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/cores/collection1/'
   [junit4]   2> 756998 T4448 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 757002 T4448 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 757007 T4448 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 757024 T4448 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 757035 T4448 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 757038 T4448 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 757044 T4448 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 757189 T4448 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 757193 T4448 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 757195 T4448 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 757214 T4448 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 757216 T4448 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 757217 T4448 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 757218 T4448 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 757218 T4448 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 757218 T4448 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 757219 T4448 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 757219 T4448 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 757219 T4448 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 757219 T4448 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 757219 T4448 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-1-001/cores/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty1/
   [junit4]   2> 757220 T4448 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@157457a3
   [junit4]   2> 757220 T4448 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty1
   [junit4]   2> 757221 T4448 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty1/index/
   [junit4]   2> 757221 T4448 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 757221 T4448 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty1/index
   [junit4]   2> 757221 T4448 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6283384548054446]
   [junit4]   2> 757222 T4448 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@385185da lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@211f69b),segFN=segments_1,generation=1}
   [junit4]   2> 757222 T4448 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 757224 T4448 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 757224 T4448 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 757225 T4448 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 757225 T4448 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 757225 T4448 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 757225 T4448 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 757225 T4448 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 757226 T4448 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 757226 T4448 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 757226 T4448 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 757227 T4448 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 757227 T4448 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 757227 T4448 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 757227 T4448 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 757228 T4448 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 757228 T4448 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 757228 T4448 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 757229 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 757229 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 757229 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 757230 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 757230 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 757230 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 757231 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 757231 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 757231 T4448 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 757232 T4448 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 757233 T4448 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757234 T4448 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757235 T4448 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757236 T4448 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757237 T4448 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 757243 T4448 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 757243 T4448 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 757244 T4448 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=58.2373046875, floorSegmentMB=1.5234375, forceMergeDeletesPctAllowed=9.088801577506036, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24295251126522147
   [junit4]   2> 757245 T4448 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@385185da lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@211f69b),segFN=segments_1,generation=1}
   [junit4]   2> 757245 T4448 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 757245 T4448 oass.SolrIndexSearcher.<init> Opening Searcher@35e1268e[collection1] main
   [junit4]   2> 757246 T4448 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 757248 T4448 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 757248 T4448 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 757248 T4448 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 757248 T4448 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 757249 T4448 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 757249 T4448 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 757249 T4448 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 757249 T4448 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 757249 T4448 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 757250 T4448 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 757250 T4452 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:46408/v_fry/eq collection:collection1 shard:shard2
   [junit4]   2> 757251 T4391 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 757251 T4391 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 757260 T4449 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@35e1268e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 757260 T4452 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 757273 T4452 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 757274 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 757274 T4452 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 757275 T4452 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2159 name=collection1 org.apache.solr.core.SolrCore@2bb2ab1c url=http://127.0.0.1:46408/v_fry/eq/collection1 node=127.0.0.1:46408_v_fry%2Feq C2159_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:46408/v_fry/eq, node_name=127.0.0.1:46408_v_fry%2Feq, state=down}
   [junit4]   2> 757275 T4452 C2159 P46408 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:46408/v_fry/eq/collection1/
   [junit4]   2> 757275 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 757275 T4452 C2159 P46408 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 757275 T4452 C2159 P46408 oasc.SyncStrategy.syncToMe http://127.0.0.1:46408/v_fry/eq/collection1/ has no replicas
   [junit4]   2> 757275 T4452 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:46408/v_fry/eq/collection1/ shard2
   [junit4]   2> 757276 T4452 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 757280 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 757281 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46408/v_fry/eq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 757384 T4447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 757384 T4430 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 757384 T4419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 757432 T4452 oasc.ZkController.register We are http://127.0.0.1:46408/v_fry/eq/collection1/ and leader is http://127.0.0.1:46408/v_fry/eq/collection1/
   [junit4]   2> 757432 T4452 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:46408/v_fry/eq
   [junit4]   2> 757432 T4452 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 757432 T4452 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 757432 T4452 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 757433 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 757434 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46408/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:46408_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 757435 T4420 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46408/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:46408_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 757537 T4447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 757538 T4430 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 757539 T4419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 757627 T4391 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/cores/collection1
   [junit4]   2> 757629 T4391 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 757631 T4391 oejs.Server.doStart jetty-9.2.6.v20141205
   [junit4]   2> 757635 T4391 oejsh.ContextHandler.doStart Started o.e.j.s.ServletContextHandler@1f28dfa5{/v_fry/eq,null,AVAILABLE}
   [junit4]   2> 757636 T4391 oejs.AbstractConnector.doStart Started ServerConnector@5f60ad53{HTTP/1.1}{127.0.0.1:37971}
   [junit4]   2> 757636 T4391 oejs.Server.doStart Started @758895ms
   [junit4]   2> 757637 T4391 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@4e0e2f2a
   [junit4]   2> 757637 T4391 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 757637 T4391 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001
   [junit4]   2> 757638 T4391 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/'
   [junit4]   2> 757665 T4391 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/solr.xml
   [junit4]   2> 757670 T4391 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/cores
   [junit4]   2> 757670 T4391 oasc.CoreContainer.<init> New CoreContainer 822388497
   [junit4]   2> 757671 T4391 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/]
   [junit4]   2> 757671 T4391 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 757672 T4391 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 757672 T4391 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 757672 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 757673 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 757673 T4391 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 757673 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 757674 T4391 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 757674 T4391 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 757674 T4391 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 757674 T4391 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 757675 T4391 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 757675 T4391 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 757676 T4391 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 757676 T4391 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 757676 T4391 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58208/solr
   [junit4]   2> 757677 T4391 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 757677 T4391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 757680 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 757680 T4466 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@832ebc5 name:ZooKeeperConnection Watcher:127.0.0.1:58208 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 757681 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 757681 T4391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 757684 T4391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 757688 T4469 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5421690 name:ZooKeeperConnection Watcher:127.0.0.1:58208/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 757689 T4391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 757692 T4391 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 758695 T4391 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37971_v_fry%2Feq
   [junit4]   2> 758695 T4391 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37971_v_fry%2Feq
   [junit4]   2> 758701 T4391 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 758703 T4391 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/cores
   [junit4]   2> 758705 T4391 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/cores/collection1/
   [junit4]   2> 758705 T4391 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 758715 T4470 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 758716 T4470 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 758716 T4470 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 758716 T4419 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 758717 T4420 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37971/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:37971_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 6
   [junit4]   2> 758718 T4420 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37971/v_fry/eq",
   [junit4]   2> 	  "node_name":"127.0.0.1:37971_v_fry%2Feq",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 758718 T4420 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 758718 T4420 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 758822 T4469 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 758822 T4430 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 758822 T4447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 758824 T4419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 759717 T4470 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 759717 T4470 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 759718 T4470 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 759718 T4470 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 759718 T4470 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 759719 T4470 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/cores/collection1/'
   [junit4]   2> 759731 T4470 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 759737 T4470 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 759741 T4470 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 759746 T4470 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 759793 T4470 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 759794 T4470 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 759798 T4470 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 760001 T4470 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 760004 T4470 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 760005 T4470 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 760018 T4470 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 760020 T4470 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 760022 T4470 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 760023 T4470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 760024 T4470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 760024 T4470 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 760025 T4470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 760025 T4470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 760025 T4470 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 760025 T4470 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 760026 T4470 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/shard-2-001/cores/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty2/
   [junit4]   2> 760026 T4470 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@157457a3
   [junit4]   2> 760027 T4470 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty2
   [junit4]   2> 760027 T4470 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty2/index/
   [junit4]   2> 760027 T4470 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 760027 T4470 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001/tempDir-001/jetty2/index
   [junit4]   2> 760028 T4470 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6283384548054446]
   [junit4]   2> 760029 T4470 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@486ee651 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@719be279),segFN=segments_1,generation=1}
   [junit4]   2> 760029 T4470 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 760032 T4470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 760032 T4470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 760032 T4470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 760033 T4470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 760033 T4470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 760033 T4470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 760033 T4470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 760034 T4470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 760034 T4470 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 760034 T4470 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 760035 T4470 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 760035 T4470 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 760036 T4470 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 760036 T4470 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 760036 T4470 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 760037 T4470 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 760037 T4470 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 760038 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 760038 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 760039 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 760039 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 760040 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 760040 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 760040 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 760041 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 760041 T4470 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 760041 T4470 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 760044 T4470 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 760045 T4470 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 760045 T4470 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 760046 T4470 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 760047 T4470 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 760064 T4470 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 760064 T4470 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 760073 T4470 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=58.2373046875, floorSegmentMB=1.5234375, forceMergeDeletesPctAllowed=9.088801577506036, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24295251126522147
   [junit4]   2> 760074 T4470 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@486ee651 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@719be279),segFN=segments_1,generation=1}
   [junit4]   2> 760074 T4470 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 760074 T4470 oass.SolrIndexSearcher.<init> Opening Searcher@18015932[collection1] main
   [junit4]   2> 760074 T4470 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 760076 T4470 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 760076 T4470 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 760076 T4470 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 760077 T4470 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 760077 T4470 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 760078 T4470 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 760078 T4470 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 760078 T4470 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 760078 T4470 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 760079 T4471 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@18015932[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 760079 T4470 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 760083 T4391 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 760084 T4391 oass.SolrDispatchFilter.init SolrDi

[...truncated too long message...]

readPoolExecutor.java:2047)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
   [junit4]   2> 	at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:259)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 779836 T4552 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher  java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@7c45600a rejected from java.util.concurrent.ThreadPoolExecutor@359e0d0b[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
   [junit4]   2> 	at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:259)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 779836 T4552 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher  java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@495adb8 rejected from java.util.concurrent.ThreadPoolExecutor@359e0d0b[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
   [junit4]   2> 	at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:259)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 779836 T4552 oaz.ClientCnxn$EventThread.processEvent ERROR Error while calling watcher  java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@47211dbf rejected from java.util.concurrent.ThreadPoolExecutor@359e0d0b[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
   [junit4]   2> 	at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:259)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZkTest -Dtests.method=test -Dtests.seed=CA15EC54DE23A8E3 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ko -Dtests.timezone=Asia/Macao -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 27.3s J1 | BasicDistributedZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: commitWithin did not work on node: http://127.0.0.1:46408/v_fry/eq/collection1 expected:<68> but was:<67>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CA15EC54DE23A8E3:4241D38E70DFC51B]:0)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:343)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 779851 T4391 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.BasicDistributedZkTest CA15EC54DE23A8E3-001
   [junit4]   2> 27260 T4390 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ko, timezone=Asia/Macao
   [junit4]   2> NOTE: Linux 3.13.0-45-generic amd64/Oracle Corporation 1.8.0_40-ea (64-bit)/cpus=8,threads=1,free=149449560,total=250007552
   [junit4]   2> NOTE: All tests run in this JVM: [TestConfigSets, TestCodecSupport, BlockCacheTest, TestReqParamsAPI, TestDynamicFieldCollectionResource, TermsComponentTest, BasicZkTest, TestLuceneMatchVersion, TestFiltering, TermVectorComponentDistributedTest, DocExpirationUpdateProcessorFactoryTest, TestHighFrequencyDictionaryFactory, DistributedExpandComponentTest, TestConfigOverlay, TestManagedStopFilterFactory, AliasIntegrationTest, TestSolrDeletionPolicy2, DistributedFacetPivotSmallAdvancedTest, TestTrie, UpdateRequestProcessorFactoryTest, SpellingQueryConverterTest, SolrXmlInZkTest, TestZkChroot, TestQuerySenderNoQuery, TestNRTOpen, SchemaVersionSpecificBehaviorTest, TestFoldingMultitermQuery, TestFieldTypeCollectionResource, TestLRUStatsCache, TestMissingGroups, StressHdfsTest, ExternalFileFieldSortTest, TestMaxScoreQueryParser, JsonLoaderTest, DistributedQueryComponentOptimizationTest, ZkNodePropsTest, TestRestManager, ConnectionManagerTest, TestRawResponseWriter, TestConfig, HdfsLockFactoryTest, LeaderElectionIntegrationTest, TestLMJelinekMercerSimilarityFactory, AutoCommitTest, TestDFRSimilarityFactory, SuggesterTSTTest, ZkStateWriterTest, EnumFieldTest, TestClusterStateMutator, StatelessScriptUpdateProcessorFactoryTest, ReturnFieldsTest, LeaderFailoverAfterPartitionTest, HdfsChaosMonkeySafeLeaderTest, TestManagedSchemaDynamicFieldResource, DirectUpdateHandlerOptimizeTest, TestOverriddenPrefixQueryForCustomFieldType, TestSearchPerf, TestSolrDeletionPolicy1, TestPHPSerializedResponseWriter, SignatureUpdateProcessorFactoryTest, TestRecoveryHdfs, SSLMigrationTest, TestPseudoReturnFields, HighlighterConfigTest, MinimalSchemaTest, DateMathParserTest, MoreLikeThisHandlerTest, TestNonDefinedSimilarityFactory, EchoParamsTest, HighlighterTest, BasicFunctionalityTest, TestSuggestSpellingConverter, DocValuesMissingTest, TestMergePolicyConfig, BadIndexSchemaTest, TestRandomFaceting, TestFreeTextSuggestions, DistributedFacetPivotSmallTest, StandardRequestHandlerTest, DistanceUnitsTest, TestRequestStatusCollectionAPI, FullSolrCloudDistribCmdsTest, LoggingHandlerTest, TestReversedWildcardFilterFactory, CloudExitableDirectoryReaderTest, BinaryUpdateRequestHandlerTest, OpenExchangeRatesOrgProviderTest, RAMDirectoryFactoryTest, TestCollationFieldDocValues, DistributedTermsComponentTest, AssignTest, TestCollectionAPI, MBeansHandlerTest, TestFastWriter, TestDocSet, TestRealTimeGet, RequestLoggingTest, MultiThreadedOCPTest, SolrCmdDistributorTest, CoreMergeIndexesAdminHandlerTest, TestAddFieldRealTimeGet, TestExactStatsCache, DebugComponentTest, TestQuerySenderListener, TestSolrCoreProperties, FacetPivotSmallTest, ShardRoutingCustomTest, IndexBasedSpellCheckerTest, TestFieldResource, TestLMDirichletSimilarityFactory, TestLeaderElectionZkExpiry, FieldMutatingUpdateProcessorTest, DeleteShardTest, TestSolrQueryParser, BasicDistributedZkTest]
   [junit4] Completed on J1 in 28.23s, 1 test, 1 failure <<< FAILURES!

[...truncated 814 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:519: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:61: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:191: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:509: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1349: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:957: There were test failures: 466 suites, 1860 tests, 1 failure, 46 ignored (21 assumptions)

Total time: 62 minutes 18 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0_40-ea-b22 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any