You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/12/21 08:29:57 UTC

[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_40-ea-b09) - Build # 11782 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/11782/
Java: 32bit/jdk1.8.0_40-ea-b09 -client -XX:+UseSerialGC (asserts: true)

1 tests failed.
FAILED:  org.apache.solr.core.TestDynamicLoading.testDistribSearch

Error Message:
{   "responseHeader":{     "status":404,     "QTime":10},   "error":{     "msg":"no such blob or version available: test/1",     "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":404,
    "QTime":10},
  "error":{
    "msg":"no such blob or version available: test/1",
    "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>
	at __randomizedtesting.SeedInfo.seed([FB2ACA74E47750AA:7ACC446C93283096]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:101)
	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:64)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9459 lines...]
   [junit4] Suite: org.apache.solr.core.TestDynamicLoading
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/init-core-data-001
   [junit4]   2> 1462604 T2975 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1462605 T2975 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1462607 T2975 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1462608 T2975 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1462608 T2976 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1462708 T2975 oasc.ZkTestServer.run start zk server on port:49014
   [junit4]   2> 1462709 T2975 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1462710 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1462714 T2983 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a73e5c name:ZooKeeperConnection Watcher:127.0.0.1:49014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1462715 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1462715 T2975 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1462715 T2975 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1462722 T2975 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1462723 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1462725 T2986 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d0665 name:ZooKeeperConnection Watcher:127.0.0.1:49014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1462726 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1462726 T2975 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1462726 T2975 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1462729 T2975 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1462732 T2975 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1462733 T2975 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1462736 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1462736 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1462740 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1462740 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1462742 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1462743 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1462745 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1462746 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1462748 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1462748 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1462750 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1462751 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1462753 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1462753 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1462755 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1462755 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1462757 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1462757 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1462759 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1462760 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1462762 T2975 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1462762 T2975 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1463084 T2975 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1463102 T2975 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:58538
   [junit4]   2> 1463104 T2975 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 1463105 T2975 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1463105 T2975 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002
   [junit4]   2> 1463106 T2975 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002/'
   [junit4]   2> 1463126 T2975 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002/solr.xml
   [junit4]   2> 1463140 T2975 oasc.CoreContainer.<init> New CoreContainer 27762515
   [junit4]   2> 1463140 T2975 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002/]
   [junit4]   2> 1463141 T2975 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1463141 T2975 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1463142 T2975 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1463142 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1463142 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1463143 T2975 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1463143 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1463143 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1463144 T2975 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1463144 T2975 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1463145 T2975 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1463145 T2975 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1463146 T2975 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1463146 T2975 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1463146 T2975 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49014/solr
   [junit4]   2> 1463147 T2975 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1463147 T2975 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1463148 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1463152 T2998 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9699b4 name:ZooKeeperConnection Watcher:127.0.0.1:49014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1463152 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1463153 T2975 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1463156 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1463158 T3001 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c0cd92 name:ZooKeeperConnection Watcher:127.0.0.1:49014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1463158 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1463160 T2975 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1463163 T2975 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1463166 T2975 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1463169 T2975 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1463172 T2975 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1463175 T2975 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1463177 T2975 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58538_
   [junit4]   2> 1463178 T2975 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58538_
   [junit4]   2> 1463181 T2975 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1463183 T2975 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1463185 T2975 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1463187 T2975 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:58538_
   [junit4]   2> 1463187 T2975 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1463189 T2975 oasc.Overseer.start Overseer (id=93005079741202435-127.0.0.1:58538_-n_0000000000) starting
   [junit4]   2> 1463191 T2975 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1463197 T2975 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1463198 T3003 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1463198 T2975 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1463200 T2975 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1463202 T2975 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1463205 T3002 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1463207 T3005 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1463207 T3005 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1463208 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1463208 T3005 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1463212 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58538",
   [junit4]   2> 	  "node_name":"127.0.0.1:58538_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1463213 T3002 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58538",
   [junit4]   2> 	  "node_name":"127.0.0.1:58538_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1463213 T3002 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1463213 T3002 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1463215 T3001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1464209 T3005 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1464209 T3005 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1464210 T3005 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1464211 T3005 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1464212 T3005 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1464213 T3005 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002/collection1/'
   [junit4]   2> 1464214 T3005 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1464214 T3005 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1464215 T3005 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1464238 T3005 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1464240 T3005 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1464252 T3005 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 1464281 T3005 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1464283 T3005 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1464295 T3005 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1464412 T3005 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1464417 T3005 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1464418 T3005 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1464433 T3005 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1464436 T3005 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1464438 T3005 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1464439 T3005 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1464440 T3005 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1464440 T3005 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1464441 T3005 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1464441 T3005 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1464441 T3005 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1464441 T3005 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1464442 T3005 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/control/data/
   [junit4]   2> 1464442 T3005 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e33fd9
   [junit4]   2> 1464443 T3005 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/control/data
   [junit4]   2> 1464443 T3005 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/control/data/index/
   [junit4]   2> 1464443 T3005 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1464443 T3005 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/control/data/index
   [junit4]   2> 1464444 T3005 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=546262122, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1464445 T3005 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@fffb93 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1fc7ab5),segFN=segments_1,generation=1}
   [junit4]   2> 1464445 T3005 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1464450 T3005 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1464450 T3005 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1464451 T3005 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1464451 T3005 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1464451 T3005 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1464451 T3005 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1464452 T3005 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1464452 T3005 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1464452 T3005 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1464453 T3005 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1464454 T3005 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1464455 T3005 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1464456 T3005 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1464456 T3005 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1464457 T3005 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1464458 T3005 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1464459 T3005 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1464459 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1464461 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1464462 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1464462 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1464463 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1464464 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1464464 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1464465 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1464466 T3005 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1464466 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1464467 T3005 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1464469 T3005 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1464470 T3005 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1464471 T3005 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1464472 T3005 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1464473 T3005 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1464474 T3005 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1464474 T3005 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=287382465, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1464475 T3005 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@fffb93 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1fc7ab5),segFN=segments_1,generation=1}
   [junit4]   2> 1464475 T3005 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1464476 T3005 oass.SolrIndexSearcher.<init> Opening Searcher@1800f12[collection1] main
   [junit4]   2> 1464476 T3005 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1464478 T3005 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1464478 T3005 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1464478 T3005 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1464479 T3005 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1464479 T3005 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1464479 T3005 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1464480 T3005 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1464480 T3005 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
   [junit4]   2> 1464483 T3005 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1464483 T3005 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1464484 T3005 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1464484 T3005 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1464485 T3006 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1800f12[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1464485 T3005 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1464487 T3009 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58538 collection:control_collection shard:shard1
   [junit4]   2> 1464487 T2975 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1464488 T2975 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1464488 T3009 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1464488 T2975 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1464489 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1464491 T3012 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1514054 name:ZooKeeperConnection Watcher:127.0.0.1:49014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1464492 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1464492 T2975 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1464494 T2975 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1464495 T3009 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1464496 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1464497 T3009 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1464497 T2975 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1464497 T3009 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1464497 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C2640 name=collection1 org.apache.solr.core.SolrCore@19e62f3 url=http://127.0.0.1:58538/collection1 node=127.0.0.1:58538_ C2640_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:58538, node_name=127.0.0.1:58538_, state=down}
   [junit4]   2> 1464498 T3009 C2640 P58538 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:58538/collection1/
   [junit4]   2> 1464499 T3009 C2640 P58538 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1464499 T3009 C2640 P58538 oasc.SyncStrategy.syncToMe http://127.0.0.1:58538/collection1/ has no replicas
   [junit4]   2> 1464500 T3009 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:58538/collection1/ shard1
   [junit4]   2> 1464500 T3009 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1464505 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1464506 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58538",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 1464609 T3001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1464610 T3012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1464656 T3009 oasc.ZkController.register We are http://127.0.0.1:58538/collection1/ and leader is http://127.0.0.1:58538/collection1/
   [junit4]   2> 1464657 T3009 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58538
   [junit4]   2> 1464657 T3009 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1464657 T3009 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1464658 T3009 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1464659 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1464661 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58538",
   [junit4]   2> 	  "node_name":"127.0.0.1:58538_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 2
   [junit4]   2> 1464662 T3002 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58538",
   [junit4]   2> 	  "node_name":"127.0.0.1:58538_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1464765 T3012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1464765 T3001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1464814 T2975 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1464815 T2975 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1464817 T2975 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54278
   [junit4]   2> 1464818 T2975 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 1464819 T2975 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1464819 T2975 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003
   [junit4]   2> 1464819 T2975 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003/'
   [junit4]   2> 1464836 T2975 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003/solr.xml
   [junit4]   2> 1464847 T2975 oasc.CoreContainer.<init> New CoreContainer 7241150
   [junit4]   2> 1464848 T2975 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003/]
   [junit4]   2> 1464849 T2975 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1464849 T2975 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1464849 T2975 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1464849 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1464850 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1464850 T2975 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1464850 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1464851 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1464851 T2975 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1464851 T2975 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1464852 T2975 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1464852 T2975 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1464853 T2975 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1464853 T2975 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1464853 T2975 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49014/solr
   [junit4]   2> 1464854 T2975 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1464854 T2975 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1464855 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1464857 T3024 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14ecb8d name:ZooKeeperConnection Watcher:127.0.0.1:49014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1464857 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1464858 T2975 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1464860 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1464861 T3027 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@793eae name:ZooKeeperConnection Watcher:127.0.0.1:49014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1464861 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1464870 T2975 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1465873 T2975 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54278_
   [junit4]   2> 1465874 T2975 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54278_
   [junit4]   2> 1465879 T2975 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1465884 T3028 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1465884 T3028 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1465886 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1465886 T3028 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1465888 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54278",
   [junit4]   2> 	  "node_name":"127.0.0.1:54278_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 1465889 T3002 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54278",
   [junit4]   2> 	  "node_name":"127.0.0.1:54278_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1465889 T3002 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1465890 T3002 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1465995 T3027 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1465995 T3001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1465995 T3012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1466886 T3028 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1466887 T3028 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1466887 T3028 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1466888 T3028 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1466889 T3028 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1466889 T3028 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003/collection1/'
   [junit4]   2> 1466890 T3028 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1466890 T3028 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1466891 T3028 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1466910 T3028 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1466912 T3028 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1466925 T3028 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 1466939 T3028 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1466940 T3028 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1466948 T3028 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1467088 T3028 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1467093 T3028 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1467095 T3028 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1467109 T3028 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1467112 T3028 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1467116 T3028 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1467117 T3028 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1467117 T3028 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1467117 T3028 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1467118 T3028 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1467118 T3028 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1467119 T3028 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1467119 T3028 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1467119 T3028 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty1/
   [junit4]   2> 1467120 T3028 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e33fd9
   [junit4]   2> 1467121 T3028 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty1
   [junit4]   2> 1467121 T3028 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty1/index/
   [junit4]   2> 1467121 T3028 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1467122 T3028 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty1/index
   [junit4]   2> 1467123 T3028 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=546262122, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1467124 T3028 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@23ee0f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4eb984),segFN=segments_1,generation=1}
   [junit4]   2> 1467125 T3028 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1467132 T3028 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1467133 T3028 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1467133 T3028 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1467134 T3028 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1467134 T3028 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1467134 T3028 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1467135 T3028 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1467135 T3028 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1467135 T3028 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1467137 T3028 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1467138 T3028 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1467139 T3028 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1467139 T3028 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1467140 T3028 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1467141 T3028 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1467143 T3028 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1467144 T3028 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1467145 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1467146 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1467148 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1467149 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1467150 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1467151 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1467153 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1467154 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1467155 T3028 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1467156 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1467157 T3028 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1467160 T3028 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1467162 T3028 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1467163 T3028 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1467165 T3028 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1467167 T3028 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1467167 T3028 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1467169 T3028 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=287382465, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1467169 T3028 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@23ee0f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4eb984),segFN=segments_1,generation=1}
   [junit4]   2> 1467170 T3028 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1467170 T3028 oass.SolrIndexSearcher.<init> Opening Searcher@161cfb9[collection1] main
   [junit4]   2> 1467170 T3028 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1467171 T3028 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1467171 T3028 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1467172 T3028 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1467172 T3028 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1467172 T3028 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1467173 T3028 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1467173 T3028 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1467173 T3028 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 1467173 T3028 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1467174 T3028 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1467175 T3029 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@161cfb9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1467177 T3028 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1467177 T3032 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54278 collection:collection1 shard:shard2
   [junit4]   2> 1467178 T2975 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1467178 T2975 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1467178 T3032 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1467183 T3032 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1467185 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1467185 T3032 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1467185 T3032 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1467185 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> ASYNC  NEW_CORE C2641 name=collection1 org.apache.solr.core.SolrCore@1aa7070 url=http://127.0.0.1:54278/collection1 node=127.0.0.1:54278_ C2641_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:54278, node_name=127.0.0.1:54278_, state=down}
   [junit4]   2> 1467185 T3032 C2641 P54278 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54278/collection1/
   [junit4]   2> 1467186 T3032 C2641 P54278 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1467186 T3032 C2641 P54278 oasc.SyncStrategy.syncToMe http://127.0.0.1:54278/collection1/ has no replicas
   [junit4]   2> 1467187 T3032 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54278/collection1/ shard2
   [junit4]   2> 1467187 T3032 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1467192 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1467194 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54278",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 1467298 T3012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1467298 T3027 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1467298 T3001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1467345 T3032 oasc.ZkController.register We are http://127.0.0.1:54278/collection1/ and leader is http://127.0.0.1:54278/collection1/
   [junit4]   2> 1467345 T3032 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54278
   [junit4]   2> 1467345 T3032 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1467346 T3032 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1467346 T3032 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1467347 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1467348 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54278",
   [junit4]   2> 	  "node_name":"127.0.0.1:54278_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1467349 T3002 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54278",
   [junit4]   2> 	  "node_name":"127.0.0.1:54278_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1467452 T3027 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1467452 T3001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1467452 T3012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1467488 T2975 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1467490 T2975 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1467492 T2975 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:44386
   [junit4]   2> 1467493 T2975 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 1467493 T2975 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1467494 T2975 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004
   [junit4]   2> 1467494 T2975 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004/'
   [junit4]   2> 1467510 T2975 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004/solr.xml
   [junit4]   2> 1467533 T2975 oasc.CoreContainer.<init> New CoreContainer 9258035
   [junit4]   2> 1467534 T2975 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004/]
   [junit4]   2> 1467534 T2975 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1467535 T2975 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1467535 T2975 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1467536 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1467536 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1467536 T2975 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1467537 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1467537 T2975 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1467538 T2975 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1467538 T2975 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1467539 T2975 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1467539 T2975 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1467539 T2975 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1467540 T2975 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1467540 T2975 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49014/solr
   [junit4]   2> 1467541 T2975 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1467541 T2975 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1467542 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1467545 T3044 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@893fd9 name:ZooKeeperConnection Watcher:127.0.0.1:49014 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1467546 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1467547 T2975 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1467550 T2975 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1467552 T3047 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1dedfd9 name:ZooKeeperConnection Watcher:127.0.0.1:49014/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1467552 T2975 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1467558 T2975 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1468561 T2975 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44386_
   [junit4]   2> 1468562 T2975 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44386_
   [junit4]   2> 1468566 T2975 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1468572 T3048 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1468573 T3048 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1468574 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1468574 T3048 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1468575 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44386",
   [junit4]   2> 	  "node_name":"127.0.0.1:44386_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 6
   [junit4]   2> 1468576 T3002 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44386",
   [junit4]   2> 	  "node_name":"127.0.0.1:44386_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1468576 T3002 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 1468576 T3002 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1468679 T3027 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1468679 T3047 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1468679 T3012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1468680 T3001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1469575 T3048 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1469575 T3048 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1469576 T3048 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1469576 T3048 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1469577 T3048 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1469577 T3048 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004/collection1/'
   [junit4]   2> 1469577 T3048 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1469578 T3048 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1469578 T3048 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1469592 T3048 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1469594 T3048 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1469605 T3048 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 1469618 T3048 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1469620 T3048 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1469631 T3048 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1469767 T3048 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1469773 T3048 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1469775 T3048 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1469790 T3048 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1469793 T3048 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1469796 T3048 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1469797 T3048 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1469797 T3048 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1469798 T3048 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1469799 T3048 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1469799 T3048 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1469799 T3048 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1469799 T3048 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1469800 T3048 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty2/
   [junit4]   2> 1469800 T3048 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e33fd9
   [junit4]   2> 1469801 T3048 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty2
   [junit4]   2> 1469801 T3048 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty2/index/
   [junit4]   2> 1469802 T3048 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1469802 T3048 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty2/index
   [junit4]   2> 1469803 T3048 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=546262122, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1469803 T3048 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@d438e5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@151d974),segFN=segments_1,generation=1}
   [junit4]   2> 1469804 T3048 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1469809 T3048 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1469810 T3048 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1469810 T3048 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1469810 T3048 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1469810 T3048 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1469811 T3048 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1469811 T3048 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1469811 T3048 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1469811 T3048 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1469812 T3048 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1469813 T3048 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1469814 T3048 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1469815 T3048 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1469816 T3048 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1469817 T3048 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1469819 T3048 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1469820 T3048 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1469821 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1469823 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1469824 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1469825 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1469826 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1469827 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1469828 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1469829 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1469829 T3048 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1469830 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1469831 T3048 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1469834 T3048 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1469835 T3048 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1469836 T3048 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1469837 T3048 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1469839 T3048 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1469839 T3048 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1469841 T3048 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=287382465, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1469841 T3048 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@d438e5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@151d974),segFN=segments_1,generation=1}
   [junit4]   2> 1469841 T3048 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1469842 T3048 oass.SolrIndexSearcher.<init> Opening Searcher@e738ae[collection1] main
   [junit4]   2> 1469842 T3048 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1469843 T3048 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1469843 T3048 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1469844 T3048 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1469844 T3048 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1469844 T3048 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1469845 T3048 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1469845 T3048 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1469845 T3048 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 1469846 T3048 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1469846 T3048 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1469847 T3049 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@e738ae[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1469848 T3048 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1469848 T3052 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:44386 collection:collection1 shard:shard1
   [junit4]   2> 1469849 T2975 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1469849 T2975 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1469849 T3052 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1469853 T3052 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1469855 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1469856 T3052 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1469856 T3002 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> 1469856 T3052 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2642 name=collection1 org.apache.solr.core.SolrCore@4ff068 url=http://127.0.0.1:44386/collection1 node=127.0.0.1:44386_ C2642_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:44386, node_name=127.0.0.1:44386_, state=down}
   [junit4]   2> 1469857 T3052 C2642 P44386 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:44386/collection1/
   [junit4]   2> 1469857 T3052 C2642 P44386 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1469858 T3052 C2642 P44386 oasc.SyncStrategy.syncToMe http://127.0.0.1:44386/collection1/ has no replicas
   [junit4]   2> 1469858 T3052 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:44386/collection1/ shard1
   [junit4]   2> 1469858 T3052 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1469862 T3001 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path

[...truncated too long message...]

 2> 1505031 T2975 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1505032 T3137 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33197",
   [junit4]   2> 	  "node_name":"127.0.0.1:33197_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 31
   [junit4]   2> 1505032 T2975 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1505032 T3137 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33197",
   [junit4]   2> 	  "node_name":"127.0.0.1:33197_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1505034 T3131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1505038 T2975 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1505039 T2975 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty4/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty4/index;done=false>>]
   [junit4]   2> 1505039 T2975 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty4/index
   [junit4]   2> 1505040 T2975 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty4 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty4;done=false>>]
   [junit4]   2> 1505040 T2975 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001/tempDir-001/jetty4
   [junit4]   2> 1505041 T2975 oasc.Overseer.close Overseer (id=93005079741202444-127.0.0.1:33197_-n_0000000004) closing
   [junit4]   2> 1505041 T3137 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:33197_
   [junit4]   2> 1506541 T3131 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1506542 T3089 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1506566 T2975 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1511556 T2975 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 1511558 T2975 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:49014 49014
   [junit4]   2> 1512123 T2976 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:49014 49014
   [junit4]   2> 1512124 T2976 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		10	/solr/configs/conf1
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		2	/solr/configs/.system
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDynamicLoading -Dtests.method=testDistribSearch -Dtests.seed=FB2ACA74E47750AA -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=da_DK -Dtests.timezone=America/Adak -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 49.5s J1 | TestDynamicLoading.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: {
   [junit4]    >   "responseHeader":{
   [junit4]    >     "status":404,
   [junit4]    >     "QTime":10},
   [junit4]    >   "error":{
   [junit4]    >     "msg":"no such blob or version available: test/1",
   [junit4]    >     "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FB2ACA74E47750AA:7ACC446C93283096]:0)
   [junit4]    > 	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:101)
   [junit4]    > 	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1512143 T2975 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FB2ACA74E47750AA-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {blobName=Lucene50(blocksize=128), _version_=PostingsFormat(name=Memory doPackFST= true), id=FST50, version=PostingsFormat(name=Memory doPackFST= true), timestamp=Lucene50(blocksize=128)}, docValues:{}, sim=DefaultSimilarity, locale=da_DK, timezone=America/Adak
   [junit4]   2> NOTE: Linux 3.13.0-43-generic i386/Oracle Corporation 1.8.0_40-ea (32-bit)/cpus=8,threads=1,free=173974448,total=301760512
   [junit4]   2> NOTE: All tests run in this JVM: [TestLFUCache, WordBreakSolrSpellCheckerTest, FastVectorHighlighterTest, TestComplexPhraseQParserPlugin, TestFastLRUCache, TestIntervalFaceting, DistribDocExpirationUpdateProcessorTest, BadCopyFieldTest, TestRandomFaceting, QueryParsingTest, AsyncMigrateRouteKeyTest, TestCoreDiscovery, EchoParamsTest, TestManagedSchema, TestRangeQuery, HdfsDirectoryTest, HighlighterTest, SignatureUpdateProcessorFactoryTest, TestElisionMultitermQuery, ExitableDirectoryReaderTest, TestManagedStopFilterFactory, ShardSplitTest, TestCollectionAPI, TestTrie, TestModifyConfFiles, DateRangeFieldTest, HdfsLockFactoryTest, SimplePostToolTest, TestCollapseQParserPlugin, TestZkChroot, ExternalFileFieldSortTest, MoreLikeThisHandlerTest, DirectSolrSpellCheckerTest, StatelessScriptUpdateProcessorFactoryTest, TimeZoneUtilsTest, PolyFieldTest, TestSolrQueryParserDefaultOperatorResource, TestPerFieldSimilarity, TestLuceneMatchVersion, BlockCacheTest, TestCollationFieldDocValues, TestSolrJ, DocumentBuilderTest, RegexBoostProcessorTest, TestImplicitCoreProperties, OutOfBoxZkACLAndCredentialsProvidersTest, ConnectionManagerTest, TestDefaultSearchFieldResource, OpenCloseCoreStressTest, TestInfoStreamLogging, TestSchemaNameResource, CopyFieldTest, TestFoldingMultitermQuery, VMParamsZkACLAndCredentialsProvidersTest, TestCopyFieldCollectionResource, TestRequestStatusCollectionAPI, TestLeaderElectionZkExpiry, DirectUpdateHandlerOptimizeTest, DebugComponentTest, TestConfig, RAMDirectoryFactoryTest, TestRecovery, TestSortingResponseWriter, TestReversedWildcardFilterFactory, DistributedTermsComponentTest, ModifyConfFileTest, CSVRequestHandlerTest, QueryEqualityTest, TestConfigReload, DistributedQueryComponentOptimizationTest, AnalyticsMergeStrategyTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, AutoCommitTest, TestFiltering, BasicZkTest, TriLevelCompositeIdRoutingTest, SampleTest, BlockDirectoryTest, SOLR749Test, SpellCheckCollatorTest, TestDynamicLoading]
   [junit4] Completed on J1 in 49.60s, 1 test, 1 failure <<< FAILURES!

[...truncated 791 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:519: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:61: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:187: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:510: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1349: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:956: There were test failures: 453 suites, 1850 tests, 1 failure, 32 ignored (22 assumptions)

Total time: 90 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 32bit/jdk1.8.0_40-ea-b09 -client -XX:+UseSerialGC (asserts: true)
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.9.0-ea-b34) - Build # 11786 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/11786/
Java: 32bit/jdk1.9.0-ea-b34 -client -XX:+UseSerialGC (asserts: true)

1 tests failed.
FAILED:  org.apache.solr.core.TestDynamicLoading.testDistribSearch

Error Message:
{   "responseHeader":{     "status":404,     "QTime":9},   "error":{     "msg":"no such blob or version available: test/1",     "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":404,
    "QTime":9},
  "error":{
    "msg":"no such blob or version available: test/1",
    "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>
	at __randomizedtesting.SeedInfo.seed([EF0F53BF98818A35:6EE9DDA7EFDEEA09]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:101)
	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:64)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9202 lines...]
   [junit4] Suite: org.apache.solr.core.TestDynamicLoading
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/init-core-data-001
   [junit4]   2> 1093213 T4516 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1093213 T4516 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1093216 T4516 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1093216 T4516 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1093217 T4517 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1093317 T4516 oasc.ZkTestServer.run start zk server on port:33612
   [junit4]   2> 1093317 T4516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1093319 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1093325 T4524 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@133306a name:ZooKeeperConnection Watcher:127.0.0.1:33612 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1093325 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1093326 T4516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1093327 T4516 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1093335 T4516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1093337 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1093339 T4527 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d29733 name:ZooKeeperConnection Watcher:127.0.0.1:33612/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1093339 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1093340 T4516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1093340 T4516 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1093347 T4516 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1093351 T4516 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1093354 T4516 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1093358 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1093359 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1093364 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1093365 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1093368 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1093369 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1093373 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1093373 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1093377 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1093378 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1093382 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1093382 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1093385 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1093385 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1093389 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1093390 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1093394 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1093395 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1093399 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1093400 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1093405 T4516 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1093405 T4516 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1093671 T4516 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1093674 T4516 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:37526
   [junit4]   2> 1093675 T4516 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@e2f2a
   [junit4]   2> 1093675 T4516 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1093675 T4516 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002
   [junit4]   2> 1093676 T4516 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002/'
   [junit4]   2> 1093689 T4516 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002/solr.xml
   [junit4]   2> 1093699 T4516 oasc.CoreContainer.<init> New CoreContainer 12466473
   [junit4]   2> 1093699 T4516 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002/]
   [junit4]   2> 1093700 T4516 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1093700 T4516 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1093700 T4516 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1093701 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1093701 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1093701 T4516 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1093702 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1093702 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1093702 T4516 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1093702 T4516 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1093703 T4516 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1093703 T4516 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1093703 T4516 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1093704 T4516 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1093704 T4516 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33612/solr
   [junit4]   2> 1093704 T4516 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1093705 T4516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1093705 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1093708 T4539 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8f701c name:ZooKeeperConnection Watcher:127.0.0.1:33612 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1093709 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1093709 T4516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1093713 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1093714 T4542 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b8808 name:ZooKeeperConnection Watcher:127.0.0.1:33612/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1093715 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1093717 T4516 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1093723 T4516 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1093728 T4516 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1093735 T4516 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1093740 T4516 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1093743 T4516 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1093745 T4516 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37526_
   [junit4]   2> 1093745 T4516 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37526_
   [junit4]   2> 1093748 T4516 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1093750 T4516 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1093753 T4516 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1093757 T4516 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:37526_
   [junit4]   2> 1093757 T4516 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1093761 T4516 oasc.Overseer.start Overseer (id=93009319713046531-127.0.0.1:37526_-n_0000000000) starting
   [junit4]   2> 1093766 T4516 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1093781 T4516 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1093797 T4544 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1093798 T4516 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1093803 T4516 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1093806 T4516 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1093812 T4543 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1093815 T4546 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1093815 T4546 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1093816 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1093816 T4546 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1093822 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37526",
   [junit4]   2> 	  "node_name":"127.0.0.1:37526_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1093823 T4543 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37526",
   [junit4]   2> 	  "node_name":"127.0.0.1:37526_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1093823 T4543 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1093824 T4543 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1093826 T4542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1094817 T4546 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1094818 T4546 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1094819 T4546 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1094819 T4546 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1094821 T4546 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1094822 T4546 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002/collection1/'
   [junit4]   2> 1094823 T4546 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1094823 T4546 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1094824 T4546 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1094842 T4546 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1094845 T4546 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1094856 T4546 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 1094867 T4546 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1094870 T4546 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1094877 T4546 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1095004 T4546 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1095008 T4546 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1095010 T4546 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1095025 T4546 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1095029 T4546 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1095031 T4546 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1095032 T4546 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1095033 T4546 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1095033 T4546 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1095034 T4546 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1095034 T4546 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1095035 T4546 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1095035 T4546 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1095035 T4546 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/control/data/
   [junit4]   2> 1095035 T4546 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9eb904
   [junit4]   2> 1095036 T4546 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/control/data
   [junit4]   2> 1095037 T4546 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/control/data/index/
   [junit4]   2> 1095037 T4546 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1095037 T4546 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/control/data/index
   [junit4]   2> 1095038 T4546 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=58.541015625, floorSegmentMB=1.8671875, forceMergeDeletesPctAllowed=9.057959127741897, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1095039 T4546 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@d2a397 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@42722e),segFN=segments_1,generation=1}
   [junit4]   2> 1095039 T4546 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1095041 T4546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1095041 T4546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1095042 T4546 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1095042 T4546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1095042 T4546 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1095042 T4546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1095043 T4546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1095043 T4546 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1095043 T4546 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1095043 T4546 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1095044 T4546 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1095044 T4546 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1095045 T4546 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1095045 T4546 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1095045 T4546 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1095046 T4546 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1095046 T4546 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1095047 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1095047 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1095048 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1095048 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1095049 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1095049 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1095050 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1095050 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1095050 T4546 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1095051 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1095051 T4546 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1095053 T4546 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1095054 T4546 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1095055 T4546 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1095055 T4546 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1095057 T4546 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1095057 T4546 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1095058 T4546 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4011995455405769]
   [junit4]   2> 1095058 T4546 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@d2a397 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@42722e),segFN=segments_1,generation=1}
   [junit4]   2> 1095058 T4546 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1095059 T4546 oass.SolrIndexSearcher.<init> Opening Searcher@57d18[collection1] main
   [junit4]   2> 1095059 T4546 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1095060 T4546 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1095060 T4546 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1095060 T4546 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1095061 T4546 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1095061 T4546 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1095061 T4546 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1095062 T4546 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1095062 T4546 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
   [junit4]   2> 1095064 T4546 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1095064 T4546 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1095064 T4546 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1095065 T4546 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1095065 T4546 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1095065 T4547 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57d18[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1095066 T4550 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37526 collection:control_collection shard:shard1
   [junit4]   2> 1095066 T4516 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1095067 T4516 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1095066 T4550 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1095067 T4516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1095068 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1095069 T4553 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6608d1 name:ZooKeeperConnection Watcher:127.0.0.1:33612/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1095070 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1095070 T4516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1095071 T4516 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1095072 T4550 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1095073 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1095073 T4550 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1095073 T4550 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3106 name=collection1 org.apache.solr.core.SolrCore@e562df url=http://127.0.0.1:37526/collection1 node=127.0.0.1:37526_ C3106_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:37526, node_name=127.0.0.1:37526_, state=down}
   [junit4]   2> 1095074 T4550 C3106 P37526 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37526/collection1/
   [junit4]   2> 1095074 T4516 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1095075 T4550 C3106 P37526 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1095074 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1095075 T4550 C3106 P37526 oasc.SyncStrategy.syncToMe http://127.0.0.1:37526/collection1/ has no replicas
   [junit4]   2> 1095076 T4550 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37526/collection1/ shard1
   [junit4]   2> 1095076 T4550 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1095080 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1095081 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37526",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 1095184 T4542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1095184 T4553 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1095231 T4550 oasc.ZkController.register We are http://127.0.0.1:37526/collection1/ and leader is http://127.0.0.1:37526/collection1/
   [junit4]   2> 1095231 T4550 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37526
   [junit4]   2> 1095231 T4550 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1095231 T4550 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1095231 T4550 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1095233 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1095234 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37526",
   [junit4]   2> 	  "node_name":"127.0.0.1:37526_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 2
   [junit4]   2> 1095235 T4543 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37526",
   [junit4]   2> 	  "node_name":"127.0.0.1:37526_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1095337 T4542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1095337 T4553 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1095350 T4516 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1095351 T4516 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1095352 T4516 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:43745
   [junit4]   2> 1095353 T4516 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@e2f2a
   [junit4]   2> 1095353 T4516 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1095354 T4516 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003
   [junit4]   2> 1095354 T4516 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003/'
   [junit4]   2> 1095367 T4516 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003/solr.xml
   [junit4]   2> 1095376 T4516 oasc.CoreContainer.<init> New CoreContainer 11797172
   [junit4]   2> 1095376 T4516 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003/]
   [junit4]   2> 1095377 T4516 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1095377 T4516 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1095377 T4516 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1095378 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1095378 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1095378 T4516 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1095379 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1095379 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1095379 T4516 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1095379 T4516 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1095380 T4516 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1095380 T4516 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1095381 T4516 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1095381 T4516 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1095381 T4516 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33612/solr
   [junit4]   2> 1095381 T4516 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1095382 T4516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1095383 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1095384 T4565 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@555f1b name:ZooKeeperConnection Watcher:127.0.0.1:33612 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1095385 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1095385 T4516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1095388 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1095389 T4568 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e8a43f name:ZooKeeperConnection Watcher:127.0.0.1:33612/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1095390 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1095399 T4516 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1096405 T4516 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43745_
   [junit4]   2> 1096407 T4516 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43745_
   [junit4]   2> 1096413 T4516 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1096420 T4569 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1096420 T4569 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1096422 T4569 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1096423 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1096424 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43745",
   [junit4]   2> 	  "node_name":"127.0.0.1:43745_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 1096426 T4543 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43745",
   [junit4]   2> 	  "node_name":"127.0.0.1:43745_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1096426 T4543 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1096427 T4543 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1096532 T4568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1096532 T4542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1096532 T4553 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1097422 T4569 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1097422 T4569 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1097423 T4569 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1097423 T4569 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1097424 T4569 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1097424 T4569 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003/collection1/'
   [junit4]   2> 1097425 T4569 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1097425 T4569 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1097425 T4569 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1097450 T4569 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1097452 T4569 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1097463 T4569 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 1097474 T4569 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1097475 T4569 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1097481 T4569 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1097596 T4569 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1097600 T4569 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1097601 T4569 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1097611 T4569 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1097613 T4569 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1097615 T4569 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1097616 T4569 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1097617 T4569 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1097617 T4569 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1097618 T4569 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1097618 T4569 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1097619 T4569 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1097619 T4569 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1097619 T4569 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty1/
   [junit4]   2> 1097619 T4569 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9eb904
   [junit4]   2> 1097620 T4569 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty1
   [junit4]   2> 1097620 T4569 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty1/index/
   [junit4]   2> 1097621 T4569 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1097621 T4569 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty1/index
   [junit4]   2> 1097621 T4569 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=58.541015625, floorSegmentMB=1.8671875, forceMergeDeletesPctAllowed=9.057959127741897, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1097622 T4569 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f218ad lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a31e2f),segFN=segments_1,generation=1}
   [junit4]   2> 1097623 T4569 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1097626 T4569 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1097627 T4569 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1097627 T4569 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1097627 T4569 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1097627 T4569 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1097628 T4569 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1097628 T4569 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1097628 T4569 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1097629 T4569 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1097629 T4569 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1097630 T4569 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1097631 T4569 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1097631 T4569 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1097632 T4569 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1097633 T4569 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1097634 T4569 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1097635 T4569 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1097635 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1097637 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1097638 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1097638 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1097639 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1097640 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1097640 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1097641 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1097642 T4569 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1097642 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1097643 T4569 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1097645 T4569 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1097645 T4569 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1097646 T4569 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1097647 T4569 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1097648 T4569 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1097648 T4569 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1097650 T4569 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4011995455405769]
   [junit4]   2> 1097650 T4569 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f218ad lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a31e2f),segFN=segments_1,generation=1}
   [junit4]   2> 1097650 T4569 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1097650 T4569 oass.SolrIndexSearcher.<init> Opening Searcher@7ea98f[collection1] main
   [junit4]   2> 1097651 T4569 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1097652 T4569 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1097652 T4569 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1097652 T4569 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1097652 T4569 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1097653 T4569 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1097653 T4569 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1097654 T4569 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1097654 T4569 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 1097654 T4569 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1097654 T4569 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1097655 T4570 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7ea98f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1097656 T4569 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1097656 T4573 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43745 collection:collection1 shard:shard2
   [junit4]   2> 1097656 T4516 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1097657 T4516 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1097657 T4573 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1097661 T4573 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1097662 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1097663 T4573 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1097663 T4573 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1097663 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> ASYNC  NEW_CORE C3107 name=collection1 org.apache.solr.core.SolrCore@1bd7c3a url=http://127.0.0.1:43745/collection1 node=127.0.0.1:43745_ C3107_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:43745, node_name=127.0.0.1:43745_, state=down}
   [junit4]   2> 1097663 T4573 C3107 P43745 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:43745/collection1/
   [junit4]   2> 1097664 T4573 C3107 P43745 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1097664 T4573 C3107 P43745 oasc.SyncStrategy.syncToMe http://127.0.0.1:43745/collection1/ has no replicas
   [junit4]   2> 1097664 T4573 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43745/collection1/ shard2
   [junit4]   2> 1097664 T4573 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1097667 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1097668 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43745",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 1097770 T4553 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1097770 T4568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1097770 T4542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1097818 T4573 oasc.ZkController.register We are http://127.0.0.1:43745/collection1/ and leader is http://127.0.0.1:43745/collection1/
   [junit4]   2> 1097818 T4573 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43745
   [junit4]   2> 1097818 T4573 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1097818 T4573 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1097818 T4573 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1097820 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1097820 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43745",
   [junit4]   2> 	  "node_name":"127.0.0.1:43745_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1097821 T4543 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43745",
   [junit4]   2> 	  "node_name":"127.0.0.1:43745_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1097923 T4553 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1097923 T4568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1097923 T4542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1097947 T4516 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1097948 T4516 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1097950 T4516 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:38558
   [junit4]   2> 1097951 T4516 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@e2f2a
   [junit4]   2> 1097951 T4516 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1097951 T4516 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004
   [junit4]   2> 1097952 T4516 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004/'
   [junit4]   2> 1097964 T4516 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004/solr.xml
   [junit4]   2> 1097973 T4516 oasc.CoreContainer.<init> New CoreContainer 3581250
   [junit4]   2> 1097974 T4516 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004/]
   [junit4]   2> 1097974 T4516 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1097974 T4516 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1097975 T4516 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1097975 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1097975 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1097976 T4516 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1097976 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1097976 T4516 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1097977 T4516 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1097977 T4516 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1097977 T4516 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1097978 T4516 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1097978 T4516 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1097978 T4516 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1097979 T4516 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33612/solr
   [junit4]   2> 1097979 T4516 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1097979 T4516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1097980 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1097982 T4585 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@92c298 name:ZooKeeperConnection Watcher:127.0.0.1:33612 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1097982 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1097983 T4516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1097985 T4516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1097987 T4588 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19832d4 name:ZooKeeperConnection Watcher:127.0.0.1:33612/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1097987 T4516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1097993 T4516 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1098996 T4516 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38558_
   [junit4]   2> 1098997 T4516 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38558_
   [junit4]   2> 1099001 T4516 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1099004 T4589 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1099005 T4589 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1099005 T4589 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1099005 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1099007 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38558",
   [junit4]   2> 	  "node_name":"127.0.0.1:38558_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 6
   [junit4]   2> 1099007 T4543 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38558",
   [junit4]   2> 	  "node_name":"127.0.0.1:38558_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1099007 T4543 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 1099008 T4543 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1099110 T4588 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1099110 T4568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1099110 T4553 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1099110 T4542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1100006 T4589 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1100006 T4589 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1100007 T4589 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1100007 T4589 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1100008 T4589 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1100009 T4589 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004/collection1/'
   [junit4]   2> 1100010 T4589 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1100010 T4589 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1100010 T4589 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1100030 T4589 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1100032 T4589 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1100057 T4589 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 1100074 T4589 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1100076 T4589 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1100087 T4589 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1100226 T4589 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1100232 T4589 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1100234 T4589 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1100253 T4589 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1100256 T4589 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1100258 T4589 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1100259 T4589 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1100260 T4589 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1100260 T4589 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1100261 T4589 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1100261 T4589 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1100261 T4589 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1100262 T4589 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1100262 T4589 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty2/
   [junit4]   2> 1100262 T4589 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@9eb904
   [junit4]   2> 1100263 T4589 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty2
   [junit4]   2> 1100263 T4589 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty2/index/
   [junit4]   2> 1100263 T4589 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1100264 T4589 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty2/index
   [junit4]   2> 1100264 T4589 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=58.541015625, floorSegmentMB=1.8671875, forceMergeDeletesPctAllowed=9.057959127741897, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1100265 T4589 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1ba52d7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ca9b7),segFN=segments_1,generation=1}
   [junit4]   2> 1100265 T4589 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1100268 T4589 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1100268 T4589 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1100269 T4589 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1100269 T4589 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1100269 T4589 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1100269 T4589 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1100270 T4589 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1100270 T4589 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1100270 T4589 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1100271 T4589 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1100272 T4589 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1100272 T4589 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1100273 T4589 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1100274 T4589 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1100274 T4589 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1100275 T4589 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1100276 T4589 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1100277 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1100278 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1100278 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1100279 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1100280 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1100280 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1100281 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1100282 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1100282 T4589 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1100283 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1100284 T4589 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1100285 T4589 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1100286 T4589 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1100287 T4589 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1100288 T4589 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1100289 T4589 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1100289 T4589 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1100290 T4589 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4011995455405769]
   [junit4]   2> 1100291 T4589 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1ba52d7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ca9b7),segFN=segments_1,generation=1}
   [junit4]   2> 1100291 T4589 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1100291 T4589 oass.SolrIndexSearcher.<init> Opening Searcher@e4101c[collection1] main
   [junit4]   2> 1100291 T4589 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1100292 T4589 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1100293 T4589 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1100293 T4589 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1100293 T4589 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1100294 T4589 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1100294 T4589 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1100294 T4589 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1100295 T4589 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 1100295 T4589 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1100295 T4589 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1100296 T4590 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@e4101c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1100296 T4589 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1100297 T4593 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38558 collection:collection1 shard:shard1
   [junit4]   2> 1100298 T4516 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1100299 T4516 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1100299 T4593 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1100304 T4593 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1100305 T4542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1100306 T4593 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1100306 T4593 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3108 name=collection1 org.apache.solr.core.SolrCore@e9b718 url=http://127.0.0.1:38558/collection1 node=127.0.0.1:38558_ C3108_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:38558, node_name=127.0.0.1:38558_, state=down}
   [junit4]   2> 1100306 T4593 C3108 P38558 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38558/collection1/
   [junit4]   2> 1100306 T4543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> 1100307 T4593 C3108 P38558 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1100307 T4593 C3108 P38558 oasc.SyncStrategy.syncToMe http://127.0.0.1:38558/collection1/ has no replicas
   [junit4]   2> 1100307 T4593 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38558/collection1/ shard1
   [junit4]   2> 1100308 T4593 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 110

[...truncated too long message...]

6 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-001/jetty4/index
   [junit4]   2> 1135495 T4516 oasc.SolrCore.close [.system_shard1_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@1729b2c
   [junit4]   2> 1135495 T4516 oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 1135495 T4516 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=3981,transaction_logs_total_number=1}
   [junit4]   2> 1135496 T4516 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1135496 T4516 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1135496 T4516 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1135497 T4516 oasc.SolrCore.closeSearcher [.system_shard1_replica2] Closing main searcher on request.
   [junit4]   2> 1135497 T4516 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1135498 T4516 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-006/.system_shard1_replica2/data/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-006/.system_shard1_replica2/data/index;done=false>>]
   [junit4]   2> 1135498 T4516 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-006/.system_shard1_replica2/data/index
   [junit4]   2> 1135498 T4516 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-006/.system_shard1_replica2/data [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-006/.system_shard1_replica2/data;done=false>>]
   [junit4]   2> 1135499 T4516 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001/tempDir-006/.system_shard1_replica2/data
   [junit4]   2> 1135499 T4516 oasc.Overseer.close Overseer (id=93009319713046540-127.0.0.1:42879_-n_0000000004) closing
   [junit4]   2> 1135500 T4679 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:42879_
   [junit4]   2> 1137001 T4630 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1137026 T4516 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1142015 T4516 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 1142017 T4516 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:33612 33612
   [junit4]   2> 1142589 T4517 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:33612 33612
   [junit4]   2> 1142591 T4517 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		10	/solr/configs/conf1
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		2	/solr/configs/.system
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDynamicLoading -Dtests.method=testDistribSearch -Dtests.seed=EF0F53BF98818A35 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=pt -Dtests.timezone=Etc/UCT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 49.4s J1 | TestDynamicLoading.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: {
   [junit4]    >   "responseHeader":{
   [junit4]    >     "status":404,
   [junit4]    >     "QTime":9},
   [junit4]    >   "error":{
   [junit4]    >     "msg":"no such blob or version available: test/1",
   [junit4]    >     "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EF0F53BF98818A35:6EE9DDA7EFDEEA09]:0)
   [junit4]    > 	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:101)
   [junit4]    > 	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1142605 T4516 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-EF0F53BF98818A35-001
   [junit4]   2> 49396 T4515 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=pt, timezone=Etc/UCT
   [junit4]   2> NOTE: Linux 3.13.0-43-generic i386/Oracle Corporation 1.9.0-ea (32-bit)/cpus=8,threads=1,free=111625952,total=279621632
   [junit4]   2> NOTE: All tests run in this JVM: [TestFiltering, DeleteInactiveReplicaTest, TestSystemIdResolver, TestManagedStopFilterFactory, DisMaxRequestHandlerTest, TestElisionMultitermQuery, TestJoin, TestModifyConfFiles, PreAnalyzedUpdateProcessorTest, TestReversedWildcardFilterFactory, DistribDocExpirationUpdateProcessorTest, MoreLikeThisHandlerTest, RollingRestartTest, TestFieldResource, TestJmxMonitoredMap, TestConfig, TestSearchPerf, PolyFieldTest, XsltUpdateRequestHandlerTest, TestSolrIndexConfig, ExternalFileFieldSortTest, SimpleCollectionCreateDeleteTest, TriLevelCompositeIdRoutingTest, TestRecoveryHdfs, FieldMutatingUpdateProcessorTest, VMParamsZkACLAndCredentialsProvidersTest, ScriptEngineTest, SpatialFilterTest, TestSchemaManager, URLClassifyProcessorTest, TestRTGBase, FileUtilsTest, ZkNodePropsTest, HdfsDirectoryTest, HdfsChaosMonkeySafeLeaderTest, DistributedQueryComponentOptimizationTest, TestFoldingMultitermQuery, TestDefaultSimilarityFactory, TestNRTOpen, TestSolrDynamicMBean, TestSerializedLuceneMatchVersion, ShardRoutingCustomTest, RAMDirectoryFactoryTest, TestClusterStateMutator, TestDistributedGrouping, TestDocBasedVersionConstraints, HdfsLockFactoryTest, FacetPivotSmallTest, CurrencyFieldOpenExchangeTest, FullSolrCloudDistribCmdsTest, OutOfBoxZkACLAndCredentialsProvidersTest, TermVectorComponentDistributedTest, SpellingQueryConverterTest, LeaderElectionIntegrationTest, BlockDirectoryTest, BasicFunctionalityTest, ModifyConfFileTest, TestLFUCache, IndexBasedSpellCheckerTest, TermsComponentTest, TestDocSet, TestFieldSortValues, TestMergePolicyConfig, TestTolerantSearch, DistributedQueryComponentCustomSortTest, BadCopyFieldTest, TestRequestStatusCollectionAPI, TestTrie, TestBulkSchemaAPI, PrimitiveFieldTypeTest, TestImplicitCoreProperties, ConnectionManagerTest, TestAddFieldRealTimeGet, TestSolrConfigHandler, DistributedQueueTest, SimplePostToolTest, CursorPagingTest, SuggesterTSTTest, TestRealTimeGet, TestCharFilters, HdfsCollectionsAPIDistributedZkTest, SynonymTokenizerTest, UUIDUpdateProcessorFallbackTest, TestSolrQueryParserDefaultOperatorResource, TestUniqueKeyFieldResource, SpellPossibilityIteratorTest, TestInfoStreamLogging, TestStandardQParsers, BinaryUpdateRequestHandlerTest, BlockCacheTest, TestDynamicLoading]
   [junit4] Completed on J1 in 49.94s, 1 test, 1 failure <<< FAILURES!

[...truncated 916 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:519: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:61: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:187: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:510: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1349: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:956: There were test failures: 453 suites, 1850 tests, 1 failure, 32 ignored (22 assumptions)

Total time: 86 minutes 48 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 32bit/jdk1.9.0-ea-b34 -client -XX:+UseSerialGC (asserts: true)
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.8.0_20) - Build # 11785 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/11785/
Java: 64bit/jdk1.8.0_20 -XX:-UseCompressedOops -XX:+UseG1GC (asserts: false)

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
ERROR: SolrZkClient opens=21 closes=20

Stack Trace:
java.lang.AssertionError: ERROR: SolrZkClient opens=21 closes=20
	at __randomizedtesting.SeedInfo.seed([FBE6A7454CA7C3D4]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingZkClients(SolrTestCaseJ4.java:462)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:188)
	at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:790)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
6 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=1185, name=TEST-CollectionsAPIDistributedZkTest.testDistribSearch-seed#[FBE6A7454CA7C3D4]-EventThread, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    2) Thread[id=1184, name=TEST-CollectionsAPIDistributedZkTest.testDistribSearch-seed#[FBE6A7454CA7C3D4]-SendThread(127.0.0.1:43269), state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    3) Thread[id=1283, name=zkCallback-158-thread-3, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=1312, name=zkCallback-158-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=1186, name=zkCallback-158-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=1279, name=zkCallback-158-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 
   1) Thread[id=1185, name=TEST-CollectionsAPIDistributedZkTest.testDistribSearch-seed#[FBE6A7454CA7C3D4]-EventThread, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   2) Thread[id=1184, name=TEST-CollectionsAPIDistributedZkTest.testDistribSearch-seed#[FBE6A7454CA7C3D4]-SendThread(127.0.0.1:43269), state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   3) Thread[id=1283, name=zkCallback-158-thread-3, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=1312, name=zkCallback-158-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=1186, name=zkCallback-158-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=1279, name=zkCallback-158-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([FBE6A7454CA7C3D4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1184, name=TEST-CollectionsAPIDistributedZkTest.testDistribSearch-seed#[FBE6A7454CA7C3D4]-SendThread(127.0.0.1:43269), state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)    2) Thread[id=1283, name=zkCallback-158-thread-3, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=1312, name=zkCallback-158-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=1186, name=zkCallback-158-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=1279, name=zkCallback-158-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1184, name=TEST-CollectionsAPIDistributedZkTest.testDistribSearch-seed#[FBE6A7454CA7C3D4]-SendThread(127.0.0.1:43269), state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   2) Thread[id=1283, name=zkCallback-158-thread-3, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=1312, name=zkCallback-158-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=1186, name=zkCallback-158-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=1279, name=zkCallback-158-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([FBE6A7454CA7C3D4]:0)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
expected:<...6A7454CA7C3D4-001/te[mpDir-003/solrj_test_core_props_shard1_replica1]> but was:<...6A7454CA7C3D4-001/te[stPropertyParamsForCreate-001/instanceDir-fxj]>

Stack Trace:
org.junit.ComparisonFailure: expected:<...6A7454CA7C3D4-001/te[mpDir-003/solrj_test_core_props_shard1_replica1]> but was:<...6A7454CA7C3D4-001/te[stPropertyParamsForCreate-001/instanceDir-fxj]>
	at __randomizedtesting.SeedInfo.seed([FBE6A7454CA7C3D4:7A00295D3BF8A3E8]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.checkInstanceDirs(CollectionsAPIDistributedZkTest.java:1154)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:901)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:205)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.TestDynamicLoading.testDistribSearch

Error Message:
{   "responseHeader":{     "status":404,     "QTime":8},   "error":{     "msg":"no such blob or version available: test/1",     "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":404,
    "QTime":8},
  "error":{
    "msg":"no such blob or version available: test/1",
    "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>
	at __randomizedtesting.SeedInfo.seed([FBE6A7454CA7C3D4:7A00295D3BF8A3E8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:101)
	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:64)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 8747 lines...]
   [junit4] Suite: org.apache.solr.core.TestDynamicLoading
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/init-core-data-001
   [junit4]   2> 129077 T327 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 129078 T327 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /wr/z
   [junit4]   2> 129083 T327 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 129084 T327 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 129085 T328 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 129184 T327 oasc.ZkTestServer.run start zk server on port:52557
   [junit4]   2> 129185 T327 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 129186 T327 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 129192 T335 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78d6ad35 name:ZooKeeperConnection Watcher:127.0.0.1:52557 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 129192 T327 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 129193 T327 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 129193 T327 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 129198 T327 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 129199 T327 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 129204 T338 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ddadf41 name:ZooKeeperConnection Watcher:127.0.0.1:52557/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 129207 T327 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 129207 T327 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 129208 T327 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 129214 T327 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 129216 T327 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 129217 T327 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 129220 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 129220 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 129223 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 129224 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 129226 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 129227 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 129229 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 129229 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 129231 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 129232 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 129233 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 129234 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 129235 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 129236 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 129237 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 129238 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 129239 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 129240 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 129241 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 129242 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 129243 T327 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 129244 T327 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 129511 T327 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 129516 T327 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:52950
   [junit4]   2> 129517 T327 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
   [junit4]   2> 129517 T327 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 129517 T327 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002
   [junit4]   2> 129518 T327 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002/'
   [junit4]   2> 129556 T327 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002/solr.xml
   [junit4]   2> 129577 T327 oasc.CoreContainer.<init> New CoreContainer 656282804
   [junit4]   2> 129577 T327 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002/]
   [junit4]   2> 129578 T327 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 129578 T327 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 129579 T327 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 129579 T327 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 129579 T327 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 129579 T327 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 129579 T327 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 129580 T327 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 129580 T327 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 129580 T327 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 129581 T327 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 129581 T327 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 129581 T327 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 129582 T327 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 129582 T327 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52557/solr
   [junit4]   2> 129582 T327 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 129583 T327 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 129583 T327 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 129587 T350 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b732996 name:ZooKeeperConnection Watcher:127.0.0.1:52557 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 129588 T327 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 129588 T327 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 129608 T327 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 129611 T353 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1479e6cc name:ZooKeeperConnection Watcher:127.0.0.1:52557/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 129612 T327 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 129618 T327 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 129621 T327 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 129623 T327 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 129625 T327 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 129626 T327 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 129628 T327 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 129629 T327 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52950_wr%2Fz
   [junit4]   2> 129630 T327 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52950_wr%2Fz
   [junit4]   2> 129631 T327 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 129633 T327 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 129634 T327 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 129635 T327 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:52950_wr%2Fz
   [junit4]   2> 129635 T327 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 129636 T327 oasc.Overseer.start Overseer (id=93008122995277827-127.0.0.1:52950_wr%2Fz-n_0000000000) starting
   [junit4]   2> 129638 T327 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 129643 T327 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 129644 T355 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 129644 T327 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 129646 T327 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 129648 T327 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 129650 T354 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 129662 T357 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 129662 T357 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 129664 T357 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 129674 T354 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52950/wr/z",
   [junit4]   2> 	  "node_name":"127.0.0.1:52950_wr%2Fz",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 129675 T354 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52950/wr/z",
   [junit4]   2> 	  "node_name":"127.0.0.1:52950_wr%2Fz",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 129675 T354 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 129676 T354 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 129678 T353 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 129682 T358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 130665 T357 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 130665 T357 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 130666 T357 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 130666 T357 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 130666 T357 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 130667 T357 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002/collection1/'
   [junit4]   2> 130667 T357 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 130668 T357 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 130668 T357 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 130677 T357 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 130679 T357 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 130687 T357 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 130697 T357 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 130698 T357 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 130703 T357 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 130829 T357 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 130833 T357 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 130834 T357 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 130844 T357 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 130846 T357 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 130849 T357 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 130850 T357 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 130850 T357 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 130851 T357 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 130851 T357 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 130851 T357 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 130851 T357 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 130852 T357 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 130852 T357 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/control/data/
   [junit4]   2> 130852 T357 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e73b61c
   [junit4]   2> 130853 T357 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/control/data
   [junit4]   2> 130853 T357 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/control/data/index/
   [junit4]   2> 130853 T357 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 130854 T357 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/control/data/index
   [junit4]   2> 130854 T357 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=90.005859375, floorSegmentMB=2.0283203125, forceMergeDeletesPctAllowed=13.465039572591932, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 130855 T357 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41e18541 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@377f0983),segFN=segments_1,generation=1}
   [junit4]   2> 130855 T357 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 130857 T357 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 130857 T357 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 130858 T357 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 130858 T357 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 130858 T357 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 130858 T357 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 130858 T357 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 130858 T357 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 130859 T357 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 130859 T357 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 130859 T357 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 130859 T357 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 130860 T357 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 130860 T357 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 130860 T357 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 130861 T357 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 130861 T357 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 130861 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 130862 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 130862 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 130862 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 130862 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 130863 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 130863 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 130864 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 130864 T357 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 130864 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 130865 T357 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 130867 T357 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 130868 T357 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 130869 T357 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 130870 T357 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 130872 T357 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 130873 T357 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 130873 T357 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=46.0986328125, floorSegmentMB=0.693359375, forceMergeDeletesPctAllowed=20.75320542138353, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1493157393385153
   [junit4]   2> 130874 T357 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41e18541 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@377f0983),segFN=segments_1,generation=1}
   [junit4]   2> 130874 T357 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 130874 T357 oass.SolrIndexSearcher.<init> Opening Searcher@477c5c70[collection1] main
   [junit4]   2> 130875 T357 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 130875 T357 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 130876 T357 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 130876 T357 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 130876 T357 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 130876 T357 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 130877 T357 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 130877 T357 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 130877 T357 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
   [junit4]   2> 130879 T357 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
   [junit4]   2> 130879 T357 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 130879 T357 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 130879 T357 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 130880 T359 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@477c5c70[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 130880 T357 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 130881 T327 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 130882 T327 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 130883 T327 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 130883 T362 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:52950/wr/z collection:control_collection shard:shard1
   [junit4]   2> 130883 T327 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130885 T362 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 130886 T365 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77dd2094 name:ZooKeeperConnection Watcher:127.0.0.1:52557/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130886 T327 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130887 T327 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 130888 T327 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130892 T362 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 130893 T327 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 130893 T327 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 130893 T327 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 130894 T327 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130895 T353 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 130896 T362 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 130896 T368 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b4e363e name:ZooKeeperConnection Watcher:127.0.0.1:52557/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130896 T362 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 130896 T327 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130896 T327 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> ASYNC  NEW_CORE C74 name=collection1 org.apache.solr.core.SolrCore@32e150af url=http://127.0.0.1:52950/wr/z/collection1 node=127.0.0.1:52950_wr%2Fz C74_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:52950/wr/z, node_name=127.0.0.1:52950_wr%2Fz, state=down}
   [junit4]   2> 130896 T362 C74 P52950 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52950/wr/z/collection1/
   [junit4]   2> 130897 T354 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 130898 T362 C74 P52950 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 130898 T362 C74 P52950 oasc.SyncStrategy.syncToMe http://127.0.0.1:52950/wr/z/collection1/ has no replicas
   [junit4]   2> 130898 T362 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:52950/wr/z/collection1/ shard1
   [junit4]   2> 130898 T362 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 130901 T354 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":2} current state version: 1
   [junit4]   2> 130901 T354 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 130902 T353 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 130902 T365 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 130904 T358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 130905 T354 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52950/wr/z",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 130906 T354 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 130907 T365 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 130910 T358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 130911 T365 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 130963 T362 oasc.ZkController.register We are http://127.0.0.1:52950/wr/z/collection1/ and leader is http://127.0.0.1:52950/wr/z/collection1/
   [junit4]   2> 130963 T362 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:52950/wr/z
   [junit4]   2> 130963 T362 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 130963 T362 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 130964 T362 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 130968 T353 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 130970 T354 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52950/wr/z",
   [junit4]   2> 	  "node_name":"127.0.0.1:52950_wr%2Fz",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 130970 T354 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52950/wr/z",
   [junit4]   2> 	  "node_name":"127.0.0.1:52950_wr%2Fz",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 131077 T353 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 131080 T365 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 131210 T327 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 131211 T327 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 131217 T327 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:45848
   [junit4]   2> 131218 T327 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@c387f44
   [junit4]   2> 131218 T327 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 131218 T327 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003
   [junit4]   2> 131218 T327 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003/'
   [junit4]   2> 131231 T327 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003/solr.xml
   [junit4]   2> 131249 T327 oasc.CoreContainer.<init> New CoreContainer 1046296965
   [junit4]   2> 131250 T327 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003/]
   [junit4]   2> 131251 T327 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 131256 T327 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 131256 T327 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 131256 T327 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 131256 T327 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 131257 T327 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 131257 T327 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 131257 T327 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 131258 T327 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 131258 T327 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 131258 T327 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 131259 T327 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 131259 T327 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 131259 T327 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 131261 T327 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52557/solr
   [junit4]   2> 131261 T327 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 131261 T327 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 131266 T327 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131269 T380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ab0ec13 name:ZooKeeperConnection Watcher:127.0.0.1:52557 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131270 T327 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131270 T327 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 131283 T327 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131285 T383 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30fa6b54 name:ZooKeeperConnection Watcher:127.0.0.1:52557/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131285 T327 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131289 T327 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 132293 T327 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45848_wr%2Fz
   [junit4]   2> 132293 T327 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45848_wr%2Fz
   [junit4]   2> 132297 T327 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 132309 T384 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 132309 T384 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 132311 T353 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 132312 T354 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45848/wr/z",
   [junit4]   2> 	  "node_name":"127.0.0.1:45848_wr%2Fz",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 132312 T384 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 132312 T384 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 132312 T354 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45848/wr/z",
   [junit4]   2> 	  "node_name":"127.0.0.1:45848_wr%2Fz",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 132312 T354 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 132312 T384 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 132312 T354 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 132314 T384 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 132314 T384 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 132415 T354 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 132416 T383 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 132416 T383 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 133314 T384 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 133314 T384 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 133315 T384 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 133315 T384 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 133315 T384 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 133316 T384 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003/collection1/'
   [junit4]   2> 133316 T384 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 133316 T384 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 133317 T384 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 133326 T384 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 133328 T384 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 133337 T384 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 133347 T384 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 133348 T384 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 133353 T384 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 133476 T384 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 133480 T384 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 133481 T384 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 133495 T384 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 133498 T384 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 133501 T384 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 133501 T384 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 133502 T384 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 133502 T384 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 133503 T384 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 133504 T384 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 133504 T384 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 133505 T384 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 133505 T384 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/jetty1/
   [junit4]   2> 133505 T384 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e73b61c
   [junit4]   2> 133506 T384 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/jetty1
   [junit4]   2> 133506 T384 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/jetty1/index/
   [junit4]   2> 133507 T384 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 133507 T384 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-FBE6A7454CA7C3D4-001/tempDir-001/jetty1/index
   [junit4]   2> 133507 T384 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=90.005859375, floorSegmentMB=2.0283203125, forceMergeDeletesPctAllowed=13.465039572591932, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 133511 T384 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@58084973 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@47e256d7),segFN=segments_1,generation=1}
   [junit4]   2> 133511 T384 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 133515 T384 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 133516 T384 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 133516 T384 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 133516 T384 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 133516 T384 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 133516 T384 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 133517 T384 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 133517 T384 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 133517 T384 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 133518 T384 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 133518 T384 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 133519 T384 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 133519 T384 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 133520 T384 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 133520 T384 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 133521 T384 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 133522 T384 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 133522 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 133523 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 133523 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 133524 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 133524 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 133525 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 133525 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 133526 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 133526 T384 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 133527 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 133527 T384 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 133529 T384 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 133531 T384 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 133532 T384 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 133533 T384 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 133534 T384 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 133534 T384 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 133536 T384 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=46.0986328125, floorSegmentMB=0.693359375, forceMergeDeletesPctAllowed=20.75320542138353, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1493157393385153
   [junit4]   2> 133536 T384 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@58084973 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@47e256d7),segFN=segments_1,generation=1}
   [junit4]   2> 133537 T384 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 133537 T384 oass.SolrIndexSearcher.<init> Opening Searcher@68b52d9[collection1] main
   [junit4]   2> 133537 T384 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 133538 T384 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 133538 T384 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 133539 T384 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 133539 T384 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 133539 T384 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 133540 T384 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 133540 T384 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 133540 T384 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 133540 T384 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 133542 T384 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 133543 T385 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68b52d9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 133544 T384 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 133544 T388 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45848/wr/z collection:collection1 shard:shard2
   [junit4]   2> 133545 T327 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 133545 T327 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 133546 T388 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 133550 T388 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 133551 T353 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 133552 T388 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 133552 T388 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C75 name=collection1 org.apache.solr.core.SolrCore@7b4dc34f url=http://127.0.0.1:45848/wr/z/collection1 node=127.0.0.1:45848_wr%2Fz C75_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:45848/wr/z, node_name=127.0.0.1:45848_wr%2Fz, state=down}
   [junit4]   2> 133552 T388 C75 P45848 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45848/wr/z/collection1/
   [junit4]   2> 133552 T388 C75 P45848 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 133552 T388 C75 P45848 oasc.SyncStrategy.syncToMe http://127.0.0.1:45848/wr/z/collection1/ has no replicas
   [junit4]   2> 133552 T388 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45848/wr/z/collection1/ shard2
   [junit4]   2> 133553 T388 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 133553 T354 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 133555 T354 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 133556 T383 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating

[...truncated too long message...]

nt.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    4) Thread[id=1312, name=zkCallback-158-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    5) Thread[id=1186, name=zkCallback-158-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    6) Thread[id=1279, name=zkCallback-158-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FBE6A7454CA7C3D4]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=1184, name=TEST-CollectionsAPIDistributedZkTest.testDistribSearch-seed#[FBE6A7454CA7C3D4]-SendThread(127.0.0.1:43269), state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
   [junit4]    >    2) Thread[id=1283, name=zkCallback-158-thread-3, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    3) Thread[id=1312, name=zkCallback-158-thread-4, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    4) Thread[id=1186, name=zkCallback-158-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    5) Thread[id=1279, name=zkCallback-158-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FBE6A7454CA7C3D4]:0)
   [junit4] Completed on J1 in 99.76s, 1 test, 2 failures, 2 errors <<< FAILURES!

[...truncated 1184 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:519: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:61: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:187: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:510: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1349: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:956: There were test failures: 453 suites, 1850 tests, 3 suite-level errors, 2 failures, 1220 ignored (15 assumptions)

Total time: 60 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 64bit/jdk1.8.0_20 -XX:-UseCompressedOops -XX:+UseG1GC (asserts: false)
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_20) - Build # 11784 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/11784/
Java: 32bit/jdk1.8.0_20 -client -XX:+UseSerialGC (asserts: true)

1 tests failed.
FAILED:  org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.testDistribSearch

Error Message:
There are still nodes recoverying - waited for 30 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 30 seconds
	at __randomizedtesting.SeedInfo.seed([8EC686856678E9:8168489EF23918D5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:178)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:840)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1459)
	at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.doTest(DistribDocExpirationUpdateProcessorTest.java:79)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9770 lines...]
   [junit4] Suite: org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/init-core-data-001
   [junit4]   2> 2298948 T6929 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 2298948 T6929 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /w/c
   [junit4]   2> 2298954 T6929 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 2298954 T6929 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2298955 T6930 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2299055 T6929 oasc.ZkTestServer.run start zk server on port:55077
   [junit4]   2> 2299056 T6929 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2299057 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2299061 T6937 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16e6081 name:ZooKeeperConnection Watcher:127.0.0.1:55077 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2299061 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2299062 T6929 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2299063 T6929 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2299066 T6929 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2299069 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2299070 T6940 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f44645 name:ZooKeeperConnection Watcher:127.0.0.1:55077/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2299071 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2299072 T6929 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2299072 T6929 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2299075 T6929 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2299078 T6929 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2299080 T6929 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2299083 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-doc-expire-update-processor.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2299084 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2299088 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 2299089 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2299092 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2299093 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2299096 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2299097 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2299100 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2299100 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2299103 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2299104 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2299106 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2299107 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 2299109 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2299110 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2299111 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2299112 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2299114 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2299114 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2299117 T6929 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2299119 T6929 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2299123 T6929 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2299125 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2299126 T6943 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1191606 name:ZooKeeperConnection Watcher:127.0.0.1:55077/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2299127 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2299127 T6929 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2299446 T6929 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2299450 T6929 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2299454 T6929 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:55783
   [junit4]   2> 2299456 T6929 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 2299457 T6929 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2299458 T6929 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002
   [junit4]   2> 2299459 T6929 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002/'
   [junit4]   2> 2299487 T6929 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002/solr.xml
   [junit4]   2> 2299519 T6929 oasc.CoreContainer.<init> New CoreContainer 31051047
   [junit4]   2> 2299520 T6929 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002/]
   [junit4]   2> 2299522 T6929 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2299523 T6929 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2299523 T6929 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2299524 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2299525 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 2299525 T6929 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2299526 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2299526 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2299527 T6929 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2299528 T6929 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2299530 T6929 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 2299532 T6929 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2299532 T6929 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2299533 T6929 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2299533 T6929 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55077/solr
   [junit4]   2> 2299534 T6929 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2299535 T6929 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2299535 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2299537 T6955 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@bef8c1 name:ZooKeeperConnection Watcher:127.0.0.1:55077 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2299538 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2299539 T6929 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2299543 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2299544 T6958 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16e0127 name:ZooKeeperConnection Watcher:127.0.0.1:55077/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2299544 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2299546 T6929 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2299550 T6929 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2299552 T6929 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 2299554 T6929 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 2299559 T6929 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 2299563 T6929 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2299565 T6929 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55783_w%2Fc
   [junit4]   2> 2299566 T6929 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55783_w%2Fc
   [junit4]   2> 2299569 T6929 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2299572 T6929 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2299574 T6929 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 2299576 T6929 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:55783_w%2Fc
   [junit4]   2> 2299577 T6929 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2299579 T6929 oasc.Overseer.start Overseer (id=93007071702679556-127.0.0.1:55783_w%2Fc-n_0000000000) starting
   [junit4]   2> 2299582 T6929 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2299590 T6929 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 2299592 T6960 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2299592 T6929 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2299595 T6929 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2299596 T6929 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2299599 T6959 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2299602 T6962 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 2299602 T6962 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2299603 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2299603 T6962 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2299607 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55783/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:55783_w%2Fc",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 2299608 T6959 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55783/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:55783_w%2Fc",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2299608 T6959 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 2299608 T6959 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2299609 T6958 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2300604 T6962 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2300605 T6962 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 2300606 T6962 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2300606 T6962 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2300608 T6962 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2300608 T6962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002/collection1/'
   [junit4]   2> 2300610 T6962 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 2300610 T6962 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 2300611 T6962 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 2300633 T6962 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 2300635 T6962 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 2300654 T6962 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 2300667 T6962 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2300668 T6962 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 2300699 T6962 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2300836 T6962 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2300838 T6962 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2300848 T6962 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2300855 T6962 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2300890 T6962 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 2300890 T6962 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2300890 T6962 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/control/data/
   [junit4]   2> 2300891 T6962 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 2300892 T6962 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/control/data
   [junit4]   2> 2300892 T6962 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/control/data/index/
   [junit4]   2> 2300893 T6962 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2300894 T6962 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/control/data/index
   [junit4]   2> 2300894 T6962 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1896927345, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 2300896 T6962 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@163c6e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@8ae67b),segFN=segments_1,generation=1}
   [junit4]   2> 2300896 T6962 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2300898 T6962 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-defaults"
   [junit4]   2> 2300912 T6962 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-field"
   [junit4]   2> 2300913 T6962 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-param"
   [junit4]   2> 2300913 T6962 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-field-with-param-default"
   [junit4]   2> 2300913 T6962 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "scheduled-delete" (default)
   [junit4]   2> 2300929 T6962 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "scheduled-delete" (default)
   [junit4]   2> 2300930 T6962 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2300930 T6962 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2300930 T6962 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2300931 T6962 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2300931 T6962 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 2300932 T6962 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 2300932 T6962 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 2300932 T6962 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 2300932 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 2300933 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 2300933 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 2300934 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 2300934 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 2300934 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 2300935 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 2300935 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: org.apache.solr.handler.admin.ShowFileRequestHandler
   [junit4]   2> 2300935 T6962 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 2300936 T6962 oasc.RequestHandlers.initHandlersFromConfig created /select: solr.SearchHandler
   [junit4]   2> 2300939 T6962 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2300940 T6962 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2300941 T6962 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2300943 T6962 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2300945 T6962 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2300945 T6962 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2300946 T6962 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=37.61328125, floorSegmentMB=2.0751953125, forceMergeDeletesPctAllowed=17.551940401305124, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2179893353270117
   [junit4]   2> 2300946 T6962 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@163c6e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@8ae67b),segFN=segments_1,generation=1}
   [junit4]   2> 2300947 T6962 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2300947 T6962 oass.SolrIndexSearcher.<init> Opening Searcher@99cf70[collection1] main
   [junit4]   2> 2300947 T6962 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2300948 T6962 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2300948 T6962 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2300949 T6962 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2300949 T6962 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2300949 T6962 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2300950 T6962 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2300950 T6962 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2300951 T6962 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
   [junit4]   2> 2300953 T6962 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2300953 T6962 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2300953 T6962 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2300956 T6962 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 2300957 T6963 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@99cf70[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2300957 T6962 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2300958 T6967 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:55783/w/c collection:control_collection shard:shard1
   [junit4]   2> 2300958 T6929 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 2300959 T6929 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2300959 T6967 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 2300964 T6967 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2300964 T6929 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2300966 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2300966 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2300967 T6967 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2300967 T6967 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2300967 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 2300967 T6970 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d757a3 name:ZooKeeperConnection Watcher:127.0.0.1:55077/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> ASYNC  NEW_CORE C2952 name=collection1 org.apache.solr.core.SolrCore@bec3b8 url=https://127.0.0.1:55783/w/c/collection1 node=127.0.0.1:55783_w%2Fc C2952_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=https://127.0.0.1:55783/w/c, node_name=127.0.0.1:55783_w%2Fc, state=down}
   [junit4]   2> 2300967 T6967 C2952 P55783 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:55783/w/c/collection1/
   [junit4]   2> 2300969 T6967 C2952 P55783 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2300969 T6967 C2952 P55783 oasc.SyncStrategy.syncToMe https://127.0.0.1:55783/w/c/collection1/ has no replicas
   [junit4]   2> 2300969 T6967 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:55783/w/c/collection1/ shard1
   [junit4]   2> 2300968 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2300969 T6967 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 2300970 T6929 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2300972 T6929 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2300975 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2300977 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55783/w/c",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 2300977 T6929 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2301082 T6958 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2301082 T6970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2301127 T6967 oasc.ZkController.register We are https://127.0.0.1:55783/w/c/collection1/ and leader is https://127.0.0.1:55783/w/c/collection1/
   [junit4]   2> 2301127 T6967 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:55783/w/c
   [junit4]   2> 2301127 T6967 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2301128 T6967 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 2301128 T6967 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2301129 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2301132 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55783/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:55783_w%2Fc",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 2
   [junit4]   2> 2301133 T6959 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55783/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:55783_w%2Fc",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2301237 T6958 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2301237 T6970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2301288 T6929 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 2301289 T6929 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2301292 T6929 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2301294 T6929 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:38386
   [junit4]   2> 2301296 T6929 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 2301297 T6929 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2301297 T6929 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003
   [junit4]   2> 2301298 T6929 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003/'
   [junit4]   2> 2301319 T6929 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003/solr.xml
   [junit4]   2> 2301341 T6929 oasc.CoreContainer.<init> New CoreContainer 23293443
   [junit4]   2> 2301342 T6929 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003/]
   [junit4]   2> 2301343 T6929 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2301343 T6929 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2301343 T6929 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2301344 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2301344 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 2301344 T6929 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2301345 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2301345 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2301345 T6929 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2301346 T6929 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2301347 T6929 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 2301349 T6929 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2301349 T6929 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2301349 T6929 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2301350 T6929 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55077/solr
   [junit4]   2> 2301350 T6929 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2301350 T6929 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2301351 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2301353 T6982 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18cce52 name:ZooKeeperConnection Watcher:127.0.0.1:55077 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2301353 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2301354 T6929 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2301357 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2301358 T6985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7046c1 name:ZooKeeperConnection Watcher:127.0.0.1:55077/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2301358 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2301365 T6929 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2302368 T6929 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38386_w%2Fc
   [junit4]   2> 2302370 T6929 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38386_w%2Fc
   [junit4]   2> 2302374 T6929 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 2302386 T6986 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 2302386 T6986 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2302388 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2302388 T6986 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2302389 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38386/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:38386_w%2Fc",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 2302390 T6959 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38386/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:38386_w%2Fc",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2302390 T6959 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 2302390 T6959 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 2302493 T6970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2302494 T6958 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2302494 T6985 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2303388 T6986 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2303389 T6986 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2303390 T6986 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2303391 T6986 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2303392 T6986 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2303393 T6986 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003/collection1/'
   [junit4]   2> 2303394 T6986 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 2303394 T6986 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 2303395 T6986 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 2303419 T6986 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 2303422 T6986 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 2303444 T6986 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 2303474 T6986 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2303476 T6986 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 2303501 T6986 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2303641 T6986 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2303642 T6986 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2303644 T6986 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2303648 T6986 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2303679 T6986 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2303680 T6986 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2303680 T6986 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty1/
   [junit4]   2> 2303680 T6986 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 2303681 T6986 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty1
   [junit4]   2> 2303681 T6986 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty1/index/
   [junit4]   2> 2303681 T6986 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 2303682 T6986 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty1/index
   [junit4]   2> 2303683 T6986 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1896927345, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 2303684 T6986 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@11e3aba lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bab32),segFN=segments_1,generation=1}
   [junit4]   2> 2303684 T6986 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2303685 T6986 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-defaults"
   [junit4]   2> 2303686 T6986 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-field"
   [junit4]   2> 2303686 T6986 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-param"
   [junit4]   2> 2303687 T6986 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-field-with-param-default"
   [junit4]   2> 2303687 T6986 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "scheduled-delete" (default)
   [junit4]   2> 2303688 T6986 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "scheduled-delete" (default)
   [junit4]   2> 2303689 T6986 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2303689 T6986 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2303689 T6986 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2303690 T6986 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2303690 T6986 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 2303691 T6986 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 2303691 T6986 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 2303691 T6986 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 2303692 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 2303692 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 2303692 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 2303693 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 2303693 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 2303693 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 2303694 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 2303694 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: org.apache.solr.handler.admin.ShowFileRequestHandler
   [junit4]   2> 2303694 T6986 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 2303695 T6986 oasc.RequestHandlers.initHandlersFromConfig created /select: solr.SearchHandler
   [junit4]   2> 2303698 T6986 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2303700 T6986 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2303702 T6986 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2303704 T6986 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2303706 T6986 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2303706 T6986 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2303707 T6986 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=37.61328125, floorSegmentMB=2.0751953125, forceMergeDeletesPctAllowed=17.551940401305124, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2179893353270117
   [junit4]   2> 2303707 T6986 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@11e3aba lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bab32),segFN=segments_1,generation=1}
   [junit4]   2> 2303708 T6986 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2303708 T6986 oass.SolrIndexSearcher.<init> Opening Searcher@1a33e76[collection1] main
   [junit4]   2> 2303708 T6986 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2303709 T6986 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2303710 T6986 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2303710 T6986 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2303710 T6986 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2303711 T6986 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2303712 T6986 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2303712 T6986 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2303712 T6986 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 2303713 T6986 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2303713 T6986 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 2303713 T6987 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1a33e76[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2303714 T6986 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2303714 T6991 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:38386/w/c collection:collection1 shard:shard2
   [junit4]   2> 2303715 T6929 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 2303716 T6929 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2303716 T6991 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 2303724 T6991 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 2303726 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2303726 T6991 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2303726 T6991 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2953 name=collection1 org.apache.solr.core.SolrCore@10f3f99 url=https://127.0.0.1:38386/w/c/collection1 node=127.0.0.1:38386_w%2Fc C2953_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=https://127.0.0.1:38386/w/c, node_name=127.0.0.1:38386_w%2Fc, state=down}
   [junit4]   2> 2303727 T6991 C2953 P38386 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:38386/w/c/collection1/
   [junit4]   2> 2303727 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 2303727 T6991 C2953 P38386 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2303728 T6991 C2953 P38386 oasc.SyncStrategy.syncToMe https://127.0.0.1:38386/w/c/collection1/ has no replicas
   [junit4]   2> 2303728 T6991 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:38386/w/c/collection1/ shard2
   [junit4]   2> 2303728 T6991 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 2303732 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2303733 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38386/w/c",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 2303836 T6970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2303836 T6985 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2303836 T6958 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2303884 T6991 oasc.ZkController.register We are https://127.0.0.1:38386/w/c/collection1/ and leader is https://127.0.0.1:38386/w/c/collection1/
   [junit4]   2> 2303884 T6991 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:38386/w/c
   [junit4]   2> 2303885 T6991 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2303885 T6991 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 2303885 T6991 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2303887 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2303888 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38386/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:38386_w%2Fc",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 2303889 T6959 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38386/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:38386_w%2Fc",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> ASYNC  NEW_CORE C2954 name=collection1 org.apache.solr.core.SolrCore@bec3b8 url=https://127.0.0.1:55783/w/c/collection1 node=127.0.0.1:55783_w%2Fc C2954_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=https://127.0.0.1:55783/w/c, node_name=127.0.0.1:55783_w%2Fc, state=active, leader=true}
   [junit4]   2> 2303957 T6964 C2954 P55783 oasup.DocExpirationUpdateProcessorFactory$DeleteExpiredDocsRunnable.run Begining periodic deletion of expired docs
   [junit4]   2> 2303960 T6964 C2954 P55783 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 2303960 T6964 C2954 P55783 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 2303961 T6964 C2954 P55783 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 2303961 T6964 C2954 P55783 oasup.LogUpdateProcessor.finish [collection1] {deleteByQuery={!cache=false}eXpField_tdt:[* TO 2014-12-21T15:21:33.431Z] (-1488113152486801408),commit=} 0 5
   [junit4]   2> 2303961 T6964 C2954 P55783 oasup.DocExpirationUpdateProcessorFactory$DeleteExpiredDocsRunnable.run Finished periodic deletion of expired docs
   [junit4]   2> 2303993 T6970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2303993 T6985 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2303993 T6958 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2304089 T6929 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 2304090 T6929 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2304095 T6929 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2304097 T6929 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:37661
   [junit4]   2> 2304100 T6929 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 2304100 T6929 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2304101 T6929 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004
   [junit4]   2> 2304101 T6929 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004/'
   [junit4]   2> 2304136 T6929 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004/solr.xml
   [junit4]   2> 2304168 T6929 oasc.CoreContainer.<init> New CoreContainer 16130423
   [junit4]   2> 2304169 T6929 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004/]
   [junit4]   2> 2304170 T6929 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2304170 T6929 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2304171 T6929 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2304171 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2304171 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 2304172 T6929 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2304172 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2304172 T6929 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2304173 T6929 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2304173 T6929 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2304175 T6929 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 2304177 T6929 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2304177 T6929 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2304177 T6929 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2304178 T6929 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55077/solr
   [junit4]   2> 2304178 T6929 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2304179 T6929 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2304179 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2304181 T7003 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c43823 name:ZooKeeperConnection Watcher:127.0.0.1:55077 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2304181 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2304182 T6929 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2304185 T6929 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2304187 T7006 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1037b30 name:ZooKeeperConnection Watcher:127.0.0.1:55077/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2304187 T6929 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2304194 T6929 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2305200 T6929 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37661_w%2Fc
   [junit4]   2> 2305202 T6929 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37661_w%2Fc
   [junit4]   2> 2305206 T6929 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 2305211 T7007 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 2305212 T7007 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2305213 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2305213 T7007 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2305214 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:37661/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:37661_w%2Fc",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 6
   [junit4]   2> 2305215 T6959 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"https://127.0.0.1:37661/w/c",
   [junit4]   2> 	  "node_name":"127.0.0.1:37661_w%2Fc",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 2305215 T6959 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 2305216 T6959 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2305319 T6970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2305320 T6985 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2305320 T6958 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2305319 T7006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2306213 T7007 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2306214 T7007 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2306215 T7007 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2306216 T7007 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2306217 T7007 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2306218 T7007 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004/collection1/'
   [junit4]   2> 2306219 T7007 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 2306219 T7007 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 2306220 T7007 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 2306250 T7007 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 2306253 T7007 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 2306270 T7007 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 2306286 T7007 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2306288 T7007 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 2306310 T7007 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2306485 T7007 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2306487 T7007 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2306488 T7007 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2306496 T7007 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2306522 T7007 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2306523 T7007 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2306523 T7007 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty2/
   [junit4]   2> 2306524 T7007 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 2306525 T7007 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty2
   [junit4]   2> 2306525 T7007 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty2/index/
   [junit4]   2> 2306525 T7007 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 2306526 T7007 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001/tempDir-001/jetty2/index
   [junit4]   2> 2306527 T7007 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1896927345, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 2306529 T7007 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bf3eaf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@19510f3),segFN=segments_1,generation=1}
   [junit4]   2> 2306529 T7007 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2306531 T7007 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-defaults"
   [junit4]   2> 2306532 T7007 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-field"
   [junit4]   2> 2306532 T7007 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-param"
   [junit4]   2> 2306533 T7007 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "convert-ttl-field-with-param-default"
   [junit4]   2> 2306533 T7007 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "scheduled-delete" (default)
   [junit4]   2> 2306534 T7007 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "scheduled-delete" (default)
   [junit4]   2> 2306535 T7007 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2306535 T7007 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2306536 T7007 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2306536 T7007 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 2306537 T7007 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 2306537 T7007 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 2306537 T7007 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 2306538 T7007 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 2306538 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 2306539 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 2306539 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 2306540 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 2306540 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 2306540 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 2306541 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 2306541 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: org.apache.solr.handler.admin.ShowFileRequestHandler
   [junit4]   2> 2306541 T7007 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 2306542 T7007 oasc.RequestHandlers.initHandlersFromConfig created /select: solr.SearchHandler
   [junit4]   2> 2306545 T7007 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2306546 T7007 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2306547 T7007 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2306548 T7007 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2306550 T7007 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2306551 T7007 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2306551 T7007 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=37.61328125, floorSegmentMB=2.0751953125, forceMergeDeletesPctAllowed=17.551940401305124, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2179893353270117
   [junit4]   2> 2306552 T7007 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bf3eaf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@19510f3),segFN=segments_1,generation=1}
   [junit4]   2> 2306552 T7007 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2306553 T7007 oass.SolrIndexSearcher.<init> Opening Searcher@1778796[collection1] main
   [junit4]   2> 2306553 T7007 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2306555 T7007 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2306555 T7007 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2306555 T7007 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2306556 T7007 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2306556 T7007 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2306557 T7007 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2306557 T7007 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2306557 T7007 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 2306558 T7007 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2306558 T7007 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 2306558 T7008 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1778796[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2306558 T7007 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2306559 T7012 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:37661/w/c collection:collection1 shard:shard1
   [junit4]   2> 2306559 T6929 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 2306560 T6929 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2306560 T7012 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2306563 T7012 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2306565 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2306565 T7012 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2306565 T7012 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2306566 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> ASYNC  NEW_CORE C2955 name=collection1 org.apache.solr.core.SolrCore@706b00 url=https://127.0.0.1:37661/w/c/collection1 node=127.0.0.1:37661_w%2Fc C2955_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=https://127.0.0.1:37661/w/c, node_name=127.0.0.1:37661_w%2Fc, state=down}
   [junit4]   2> 2306566 T7012 C2955 P37661 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:37661/w/c/collection1/
   [junit4]   2> 2306566 T7012 C2955 P37661 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2306566 T7012 C2955 P37661 oasc.SyncStrategy.syncToMe https://127.0.0.1:37661/w/c/collection1/ has no replicas
   [junit4]   2> 2306567 T7012 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:37661/w/c/collection1/ shard1
   [junit4]   2> 2306567 T7012 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2306571 T6958 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2306572 T6959 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:37661/w/c",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 7
   [junit4]   2> 2306675 T6970 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2306675 T7006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live no

[...truncated too long message...]

apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:331)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:328)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.orderedChildren(DistributedQueue.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.remove(DistributedQueue.java:189)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.poll(DistributedQueue.java:616)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:277)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 2374857 T7165 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:53206_w%2Fc
   [junit4]   2> 2374857 T7156 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2374857 T7051 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2374880 T6929 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/w/c,null}
   [junit4]   2> 2379869 T6929 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2379873 T6929 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:55077 55077
   [junit4]   2> 2380425 T6930 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:55077 55077
   [junit4]   2> 2380428 T6930 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistribDocExpirationUpdateProcessorTest -Dtests.method=testDistribSearch -Dtests.seed=8EC686856678E9 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en_IN -Dtests.timezone=Australia/Adelaide -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 81.5s J1 | DistribDocExpirationUpdateProcessorTest.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: There are still nodes recoverying - waited for 30 seconds
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8EC686856678E9:8168489EF23918D5]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:178)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:840)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1459)
   [junit4]    > 	at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.doTest(DistribDocExpirationUpdateProcessorTest.java:79)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2380455 T6929 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DistribDocExpirationUpdateProcessorTest-8EC686856678E9-001
   [junit4]   2> 81515 T6928 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=PostingsFormat(name=Memory doPackFST= true), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=en_IN, timezone=Australia/Adelaide
   [junit4]   2> NOTE: Linux 3.13.0-43-generic i386/Oracle Corporation 1.8.0_20 (32-bit)/cpus=8,threads=1,free=123625432,total=283115520
   [junit4]   2> NOTE: All tests run in this JVM: [QueryResultKeyTest, ExternalCollectionsTest, FieldMutatingUpdateProcessorTest, FileUtilsTest, TestSolrDeletionPolicy1, TestMergePolicyConfig, DistributedExpandComponentTest, VMParamsZkACLAndCredentialsProvidersTest, SuggesterTSTTest, TestDynamicLoading, TestAddFieldRealTimeGet, SpellPossibilityIteratorTest, TestLuceneMatchVersion, QueryEqualityTest, TestSolrDynamicMBean, TestImplicitCoreProperties, TestReload, TestCollapseQParserPlugin, TestNonDefinedSimilarityFactory, CopyFieldTest, EchoParamsTest, SolrRequestParserTest, CurrencyFieldOpenExchangeTest, TestLFUCache, RAMDirectoryFactoryTest, TestSolrConfigHandlerCloud, TestHighlightDedupGrouping, TestRandomFaceting, SpatialFilterTest, TestPostingsSolrHighlighter, DirectUpdateHandlerOptimizeTest, XsltUpdateRequestHandlerTest, DocValuesMissingTest, SpellCheckComponentTest, QueryParsingTest, FacetPivotSmallTest, TestElisionMultitermQuery, ExitableDirectoryReaderTest, TestSortingResponseWriter, FileBasedSpellCheckerTest, ChaosMonkeySafeLeaderTest, TestPHPSerializedResponseWriter, SolrXmlInZkTest, SimplePostToolTest, CurrencyFieldXmlFileTest, TestFastLRUCache, ConvertedLegacyTest, ReplicationFactorTest, HdfsCollectionsAPIDistributedZkTest, TestUtils, TestSuggestSpellingConverter, TestSolrXmlPersistence, TermsComponentTest, TestSystemIdResolver, TestManagedStopFilterFactory, FullSolrCloudDistribCmdsTest, TestSchemaNameResource, OverriddenZkACLAndCredentialsProvidersTest, SpellCheckCollatorTest, TestValueSourceCache, CSVRequestHandlerTest, TestCollationFieldDocValues, TestSolr4Spatial, RegexBoostProcessorTest, SOLR749Test, TestConfig, BlockDirectoryTest, TestPerFieldSimilarity, TestDocBasedVersionConstraints, TestFiltering, TestSerializedLuceneMatchVersion, ModifyConfFileTest, SpellingQueryConverterTest, BasicDistributedZk2Test, BadComponentTest, UpdateRequestProcessorFactoryTest, DeleteInactiveReplicaTest, DistributedFacetPivotLongTailTest, ParsingFieldUpdateProcessorsTest, PreAnalyzedUpdateProcessorTest, AnalyticsMergeStrategyTest, TestIntervalFaceting, TestJoin, SearchHandlerTest, TestInfoStreamLogging, DistributedTermsComponentTest, TestDefaultSimilarityFactory, PolyFieldTest, IndexSchemaTest, MoreLikeThisHandlerTest, BlockCacheTest, TestComplexPhraseQParserPlugin, TestRemoteStreaming, TestFieldResource, JsonLoaderTest, TestDefaultSearchFieldResource, AliasIntegrationTest, SolrCmdDistributorTest, HdfsDirectoryTest, TermVectorComponentTest, TestTolerantSearch, TestMissingGroups, TestSchemaResource, IndexBasedSpellCheckerTest, TestDistributedGrouping, LoggingHandlerTest, BasicDistributedZkTest, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, TestZkChroot, AutoCommitTest, TestHashPartitioner, ZkControllerTest, TestRealTimeGet, TestStressReorder, HardAutoCommitTest, TestCoreContainer, StatsComponentTest, PeerSyncTest, DirectUpdateHandlerTest, OverseerCollectionProcessorTest, TestIndexSearcher, ShowFileRequestHandlerTest, SolrIndexSplitterTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, CoreAdminHandlerTest, TestFoldingMultitermQuery, TestTrie, TestCSVLoader, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestUpdate, StatelessScriptUpdateProcessorFactoryTest, DebugComponentTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, StandardRequestHandlerTest, TestWriterPerf, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, PrimitiveFieldTypeTest, DocumentBuilderTest, TestSolrQueryParser, FastVectorHighlighterTest, IndexSchemaRuntimeFieldTest, ReturnFieldsTest, PingRequestHandlerTest, TestComponentsName, TestBinaryResponseWriter, TestSolrIndexConfig, TestStressRecovery, PrimUtilsTest, DOMUtilTest, TestSolrJ, TestLRUCache, ZkNodePropsTest, SystemInfoHandlerTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, AsyncMigrateRouteKeyTest, CloudExitableDirectoryReaderTest, DistribDocExpirationUpdateProcessorTest]
   [junit4] Completed on J1 in 81.82s, 1 test, 1 failure <<< FAILURES!

[...truncated 324 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:519: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:61: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:187: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:510: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1349: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:956: There were test failures: 453 suites, 1850 tests, 1 failure, 32 ignored (22 assumptions)

Total time: 95 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 32bit/jdk1.8.0_20 -client -XX:+UseSerialGC (asserts: true)
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0_40-ea-b09) - Build # 11783 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/11783/
Java: 32bit/jdk1.8.0_40-ea-b09 -client -XX:+UseSerialGC (asserts: true)

1 tests failed.
FAILED:  org.apache.solr.core.TestDynamicLoading.testDistribSearch

Error Message:
{   "responseHeader":{     "status":404,     "QTime":10},   "error":{     "msg":"no such blob or version available: test/1",     "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":404,
    "QTime":10},
  "error":{
    "msg":"no such blob or version available: test/1",
    "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>
	at __randomizedtesting.SeedInfo.seed([61151B87BC111CF3:E0F3959FCB4E7CCF]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:101)
	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:64)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:497)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 8863 lines...]
   [junit4] Suite: org.apache.solr.core.TestDynamicLoading
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/init-core-data-001
   [junit4]   2> 322041 T1782 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 322041 T1782 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_iw/nr
   [junit4]   2> 322047 T1782 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 322048 T1782 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 322049 T1783 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 322149 T1782 oasc.ZkTestServer.run start zk server on port:33501
   [junit4]   2> 322149 T1782 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 322150 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 322156 T1790 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18f0e25 name:ZooKeeperConnection Watcher:127.0.0.1:33501 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 322156 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 322156 T1782 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 322157 T1782 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 322161 T1782 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 322162 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 322164 T1793 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1726ed name:ZooKeeperConnection Watcher:127.0.0.1:33501/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 322165 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 322165 T1782 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 322166 T1782 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 322171 T1782 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 322174 T1782 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 322177 T1782 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 322181 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 322182 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 322186 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 322188 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 322192 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 322192 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 322195 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 322196 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 322200 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 322200 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 322204 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 322205 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 322208 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 322208 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 322211 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 322212 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 322216 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 322217 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 322221 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 322222 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 322226 T1782 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 322227 T1782 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 322538 T1782 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 322545 T1782 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:39626
   [junit4]   2> 322546 T1782 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 322546 T1782 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 322547 T1782 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002
   [junit4]   2> 322547 T1782 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002/'
   [junit4]   2> 322565 T1782 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002/solr.xml
   [junit4]   2> 322591 T1782 oasc.CoreContainer.<init> New CoreContainer 21505784
   [junit4]   2> 322591 T1782 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002/]
   [junit4]   2> 322592 T1782 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 322593 T1782 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 322593 T1782 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 322594 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 322594 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 322594 T1782 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 322594 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 322595 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 322595 T1782 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 322595 T1782 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 322596 T1782 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 322596 T1782 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 322597 T1782 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 322597 T1782 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 322597 T1782 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33501/solr
   [junit4]   2> 322598 T1782 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 322598 T1782 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 322599 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 322601 T1805 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18019d9 name:ZooKeeperConnection Watcher:127.0.0.1:33501 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 322602 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 322602 T1782 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 322606 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 322608 T1808 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3efdbe name:ZooKeeperConnection Watcher:127.0.0.1:33501/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 322608 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 322610 T1782 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 322614 T1782 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 322619 T1782 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 322622 T1782 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 322626 T1782 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 322629 T1782 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 322630 T1782 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39626__iw%2Fnr
   [junit4]   2> 322631 T1782 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39626__iw%2Fnr
   [junit4]   2> 322633 T1782 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 322635 T1782 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 322637 T1782 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 322640 T1782 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:39626__iw%2Fnr
   [junit4]   2> 322640 T1782 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 322643 T1782 oasc.Overseer.start Overseer (id=93006250696704003-127.0.0.1:39626__iw%2Fnr-n_0000000000) starting
   [junit4]   2> 322647 T1782 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 322653 T1782 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 322654 T1810 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 322654 T1782 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 322656 T1782 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 322657 T1782 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 322659 T1809 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 322662 T1812 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 322662 T1812 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 322663 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 322663 T1812 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 322668 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39626/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:39626__iw%2Fnr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 322669 T1809 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39626/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:39626__iw%2Fnr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 322670 T1809 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 322670 T1809 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 322672 T1808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 323664 T1812 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 323665 T1812 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 323665 T1812 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 323666 T1812 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 323667 T1812 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 323668 T1812 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002/collection1/'
   [junit4]   2> 323669 T1812 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 323670 T1812 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 323671 T1812 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 323694 T1812 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 323697 T1812 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 323723 T1812 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 323740 T1812 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 323742 T1812 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 323767 T1812 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 323958 T1812 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 323966 T1812 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 323968 T1812 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 323990 T1812 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 323995 T1812 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 324000 T1812 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 324001 T1812 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 324002 T1812 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 324002 T1812 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 324004 T1812 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 324004 T1812 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 324004 T1812 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 324005 T1812 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 324005 T1812 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-002/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/control/data/
   [junit4]   2> 324006 T1812 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c9152
   [junit4]   2> 324007 T1812 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/control/data
   [junit4]   2> 324007 T1812 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/control/data/index/
   [junit4]   2> 324008 T1812 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 324008 T1812 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/control/data/index
   [junit4]   2> 324009 T1812 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 324010 T1812 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@112ebf4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9c49e9),segFN=segments_1,generation=1}
   [junit4]   2> 324011 T1812 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 324015 T1812 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 324016 T1812 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 324016 T1812 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 324017 T1812 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 324017 T1812 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 324018 T1812 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 324018 T1812 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 324019 T1812 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 324019 T1812 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 324021 T1812 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 324022 T1812 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 324023 T1812 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 324024 T1812 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 324025 T1812 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 324026 T1812 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 324027 T1812 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 324027 T1812 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 324028 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 324029 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 324029 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 324030 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 324031 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 324031 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 324032 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 324033 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 324033 T1812 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 324034 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 324035 T1812 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 324037 T1812 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 324038 T1812 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 324039 T1812 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 324040 T1812 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 324042 T1812 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 324042 T1812 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 324043 T1812 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 324043 T1812 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@112ebf4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9c49e9),segFN=segments_1,generation=1}
   [junit4]   2> 324044 T1812 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 324044 T1812 oass.SolrIndexSearcher.<init> Opening Searcher@3c9d56[collection1] main
   [junit4]   2> 324044 T1812 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 324046 T1812 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 324046 T1812 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 324047 T1812 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 324047 T1812 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 324047 T1812 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 324048 T1812 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 324048 T1812 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 324049 T1812 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
   [junit4]   2> 324053 T1812 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
   [junit4]   2> 324054 T1812 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 324054 T1812 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 324054 T1812 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 324056 T1813 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3c9d56[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 324057 T1812 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 324058 T1816 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39626/_iw/nr collection:control_collection shard:shard1
   [junit4]   2> 324059 T1782 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 324060 T1782 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 324060 T1816 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 324061 T1782 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 324062 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 324064 T1819 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c38a2 name:ZooKeeperConnection Watcher:127.0.0.1:33501/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 324065 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 324066 T1782 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 324067 T1782 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 324069 T1816 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 324071 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 324072 T1816 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 324072 T1816 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 324072 T1782 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> ASYNC  NEW_CORE C616 name=collection1 org.apache.solr.core.SolrCore@3dbac8 url=http://127.0.0.1:39626/_iw/nr/collection1 node=127.0.0.1:39626__iw%2Fnr C616_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:39626/_iw/nr, node_name=127.0.0.1:39626__iw%2Fnr, state=down}
   [junit4]   2> 324072 T1816 C616 P39626 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39626/_iw/nr/collection1/
   [junit4]   2> 324073 T1816 C616 P39626 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 324074 T1816 C616 P39626 oasc.SyncStrategy.syncToMe http://127.0.0.1:39626/_iw/nr/collection1/ has no replicas
   [junit4]   2> 324075 T1816 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39626/_iw/nr/collection1/ shard1
   [junit4]   2> 324075 T1816 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 324076 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 324083 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 324084 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39626/_iw/nr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 324188 T1819 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 324188 T1808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 324234 T1816 oasc.ZkController.register We are http://127.0.0.1:39626/_iw/nr/collection1/ and leader is http://127.0.0.1:39626/_iw/nr/collection1/
   [junit4]   2> 324235 T1816 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39626/_iw/nr
   [junit4]   2> 324235 T1816 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 324235 T1816 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 324236 T1816 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 324237 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 324239 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39626/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:39626__iw%2Fnr",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 2
   [junit4]   2> 324240 T1809 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39626/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:39626__iw%2Fnr",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 324343 T1808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 324343 T1819 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 324395 T1782 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 324395 T1782 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 324397 T1782 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:36618
   [junit4]   2> 324398 T1782 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 324399 T1782 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 324399 T1782 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003
   [junit4]   2> 324399 T1782 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003/'
   [junit4]   2> 324415 T1782 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003/solr.xml
   [junit4]   2> 324428 T1782 oasc.CoreContainer.<init> New CoreContainer 2448470
   [junit4]   2> 324428 T1782 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003/]
   [junit4]   2> 324429 T1782 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 324429 T1782 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 324429 T1782 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 324430 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 324430 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 324430 T1782 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 324431 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 324431 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 324431 T1782 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 324432 T1782 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 324432 T1782 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 324432 T1782 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 324433 T1782 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 324433 T1782 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 324433 T1782 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33501/solr
   [junit4]   2> 324434 T1782 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 324434 T1782 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 324435 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 324438 T1831 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f91972 name:ZooKeeperConnection Watcher:127.0.0.1:33501 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 324438 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 324438 T1782 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 324442 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 324444 T1834 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4bb2f6 name:ZooKeeperConnection Watcher:127.0.0.1:33501/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 324444 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 324458 T1782 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 325463 T1782 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36618__iw%2Fnr
   [junit4]   2> 325464 T1782 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36618__iw%2Fnr
   [junit4]   2> 325468 T1782 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 325472 T1835 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 325472 T1835 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 325473 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 325473 T1835 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 325475 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36618/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:36618__iw%2Fnr",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 325477 T1809 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36618/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:36618__iw%2Fnr",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 325477 T1809 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 325478 T1809 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 325583 T1819 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 325583 T1808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 325583 T1834 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 326474 T1835 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 326475 T1835 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 326476 T1835 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 326477 T1835 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 326479 T1835 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 326479 T1835 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003/collection1/'
   [junit4]   2> 326481 T1835 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 326481 T1835 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 326482 T1835 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 326499 T1835 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 326502 T1835 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 326524 T1835 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 326547 T1835 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 326548 T1835 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 326574 T1835 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 326728 T1835 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 326734 T1835 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 326736 T1835 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 326753 T1835 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 326757 T1835 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 326760 T1835 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 326761 T1835 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 326762 T1835 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 326762 T1835 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 326763 T1835 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 326763 T1835 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 326764 T1835 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 326764 T1835 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 326764 T1835 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-003/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty1/
   [junit4]   2> 326765 T1835 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c9152
   [junit4]   2> 326766 T1835 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty1
   [junit4]   2> 326766 T1835 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty1/index/
   [junit4]   2> 326766 T1835 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 326767 T1835 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty1/index
   [junit4]   2> 326768 T1835 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 326768 T1835 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19fa8df lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@f6b162),segFN=segments_1,generation=1}
   [junit4]   2> 326769 T1835 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 326774 T1835 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 326775 T1835 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 326775 T1835 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 326776 T1835 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 326776 T1835 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 326776 T1835 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 326777 T1835 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 326777 T1835 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 326778 T1835 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 326779 T1835 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 326780 T1835 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 326781 T1835 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 326782 T1835 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 326783 T1835 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 326784 T1835 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 326786 T1835 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 326787 T1835 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 326788 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 326789 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 326790 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 326791 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 326792 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 326793 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 326794 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 326795 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 326795 T1835 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 326796 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 326802 T1835 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 326804 T1835 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 326805 T1835 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 326807 T1835 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 326809 T1835 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 326811 T1835 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 326811 T1835 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 326813 T1835 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 326813 T1835 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19fa8df lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@f6b162),segFN=segments_1,generation=1}
   [junit4]   2> 326814 T1835 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 326814 T1835 oass.SolrIndexSearcher.<init> Opening Searcher@16530c5[collection1] main
   [junit4]   2> 326814 T1835 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 326815 T1835 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 326816 T1835 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 326816 T1835 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 326816 T1835 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 326817 T1835 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 326818 T1835 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 326818 T1835 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 326818 T1835 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 326818 T1835 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 326819 T1835 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 326821 T1836 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@16530c5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 326822 T1835 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 326823 T1839 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36618/_iw/nr collection:collection1 shard:shard2
   [junit4]   2> 326823 T1782 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 326824 T1782 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 326824 T1839 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 326831 T1839 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 326833 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 326833 T1839 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 326833 T1839 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C617 name=collection1 org.apache.solr.core.SolrCore@4e6830 url=http://127.0.0.1:36618/_iw/nr/collection1 node=127.0.0.1:36618__iw%2Fnr C617_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:36618/_iw/nr, node_name=127.0.0.1:36618__iw%2Fnr, state=down}
   [junit4]   2> 326834 T1839 C617 P36618 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36618/_iw/nr/collection1/
   [junit4]   2> 326834 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 326834 T1839 C617 P36618 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 326835 T1839 C617 P36618 oasc.SyncStrategy.syncToMe http://127.0.0.1:36618/_iw/nr/collection1/ has no replicas
   [junit4]   2> 326835 T1839 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36618/_iw/nr/collection1/ shard2
   [junit4]   2> 326835 T1839 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 326840 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 326841 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36618/_iw/nr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 326944 T1819 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 326945 T1808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 326945 T1834 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 326991 T1839 oasc.ZkController.register We are http://127.0.0.1:36618/_iw/nr/collection1/ and leader is http://127.0.0.1:36618/_iw/nr/collection1/
   [junit4]   2> 326991 T1839 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36618/_iw/nr
   [junit4]   2> 326991 T1839 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 326992 T1839 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 326992 T1839 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 326993 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 326995 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36618/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:36618__iw%2Fnr",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 326995 T1809 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36618/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:36618__iw%2Fnr",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 327098 T1819 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 327098 T1808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 327098 T1834 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 327127 T1782 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 327128 T1782 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 327130 T1782 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50792
   [junit4]   2> 327131 T1782 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@387f44
   [junit4]   2> 327132 T1782 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 327132 T1782 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004
   [junit4]   2> 327132 T1782 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004/'
   [junit4]   2> 327153 T1782 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004/solr.xml
   [junit4]   2> 327180 T1782 oasc.CoreContainer.<init> New CoreContainer 28013566
   [junit4]   2> 327181 T1782 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004/]
   [junit4]   2> 327182 T1782 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 327182 T1782 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 327183 T1782 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 327183 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 327184 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 327184 T1782 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 327185 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 327185 T1782 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 327186 T1782 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 327186 T1782 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 327187 T1782 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 327187 T1782 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 327188 T1782 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 327188 T1782 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 327189 T1782 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33501/solr
   [junit4]   2> 327189 T1782 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 327190 T1782 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 327190 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 327193 T1851 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c2d6d3 name:ZooKeeperConnection Watcher:127.0.0.1:33501 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 327193 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 327194 T1782 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 327196 T1782 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 327198 T1854 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e187ef name:ZooKeeperConnection Watcher:127.0.0.1:33501/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 327199 T1782 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 327204 T1782 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 328209 T1782 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50792__iw%2Fnr
   [junit4]   2> 328211 T1782 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50792__iw%2Fnr
   [junit4]   2> 328216 T1782 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 328222 T1855 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 328223 T1855 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 328224 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 328224 T1855 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 328225 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50792/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:50792__iw%2Fnr",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 6
   [junit4]   2> 328226 T1809 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50792/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:50792__iw%2Fnr",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 328226 T1809 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 328226 T1809 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 328331 T1819 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 328332 T1808 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 328331 T1834 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 328331 T1854 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 329224 T1855 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 329225 T1855 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 329226 T1855 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 329227 T1855 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 329228 T1855 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 329229 T1855 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004/collection1/'
   [junit4]   2> 329230 T1855 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 329231 T1855 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 329232 T1855 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 329254 T1855 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 329257 T1855 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 329279 T1855 oasc.SolrConfig.<init> Using Lucene MatchVersion: 6.0.0
   [junit4]   2> 329295 T1855 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 329296 T1855 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 329310 T1855 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 329500 T1855 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 329505 T1855 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 329507 T1855 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 329521 T1855 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 329524 T1855 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 329527 T1855 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 329527 T1855 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 329528 T1855 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 329528 T1855 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 329529 T1855 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 329529 T1855 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 329529 T1855 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 329529 T1855 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 329530 T1855 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-004/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty2/
   [junit4]   2> 329530 T1855 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c9152
   [junit4]   2> 329531 T1855 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty2
   [junit4]   2> 329531 T1855 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty2/index/
   [junit4]   2> 329531 T1855 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 329532 T1855 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty2/index
   [junit4]   2> 329532 T1855 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 329533 T1855 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@baf69a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@fe4454),segFN=segments_1,generation=1}
   [junit4]   2> 329533 T1855 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 329538 T1855 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 329539 T1855 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 329539 T1855 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 329539 T1855 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 329539 T1855 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 329540 T1855 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 329540 T1855 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 329540 T1855 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 329541 T1855 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 329542 T1855 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 329542 T1855 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 329543 T1855 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 329544 T1855 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 329545 T1855 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 329545 T1855 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 329547 T1855 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 329549 T1855 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 329552 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 329553 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 329554 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 329555 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 329556 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 329557 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 329558 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 329559 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 329560 T1855 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 329561 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 329562 T1855 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 329564 T1855 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 329565 T1855 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 329566 T1855 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 329567 T1855 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 329569 T1855 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 329569 T1855 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 329571 T1855 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 329571 T1855 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@baf69a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@fe4454),segFN=segments_1,generation=1}
   [junit4]   2> 329571 T1855 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 329572 T1855 oass.SolrIndexSearcher.<init> Opening Searcher@ab82c0[collection1] main
   [junit4]   2> 329572 T1855 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 329573 T1855 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 329573 T1855 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 329574 T1855 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 329574 T1855 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 329574 T1855 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 329575 T1855 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 329575 T1855 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 329576 T1855 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 329576 T1855 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 329576 T1855 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 329578 T1856 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ab82c0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 329578 T1855 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 329579 T1859 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50792/_iw/nr collection:collection1 shard:shard1
   [junit4]   2> 329579 T1782 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 329580 T1782 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 329580 T1859 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 329584 T1859 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 329585 T1808 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 329586 T1859 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 329586 T1859 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 329586 T1809 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> ASYNC  NEW_CORE C618 name=collection1 org.apache.solr.core.SolrCore@14cb78f url=http://127.0.0.1:50792/_iw/nr/collection1 node=127.0.0.1:50792__iw%2Fnr C618_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:50792/_iw/nr, node_name=127.0.0.1:50792__iw%2Fnr, state=down}
   [junit4]   2> 329587 T1859 C618 P50792 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50792/_iw/nr/collection1/
   [junit4]   2> 329587 T1859 C618 P50792 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 329588 T1859 C618 P50792 oasc.SyncStrategy.syncToMe http://127.0.0.1:50792/_iw/nr/collection1/ has no replicas
   [junit4]   2> 329588 T1859 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50792/_iw/nr/collection1/ shard1
   [junit4] 

[...truncated too long message...]

[junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32789/_iw/nr",
   [junit4]   2> 	  "node_name":"127.0.0.1:32789__iw%2Fnr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":".system",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 364804 T1782 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 364804 T1782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty4/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty4/index;done=false>>]
   [junit4]   2> 364805 T1782 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty4/index
   [junit4]   2> 364805 T1940 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 364805 T1782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty4 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty4;done=false>>]
   [junit4]   2> 364806 T1782 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-001/jetty4
   [junit4]   2> 364806 T1782 oasc.SolrCore.close [.system_shard1_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@a06f2
   [junit4]   2> 364806 T1782 oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 364807 T1782 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=3987,transaction_logs_total_number=1}
   [junit4]   2> 364807 T1782 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 364807 T1782 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 364808 T1782 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 364809 T1782 oasc.SolrCore.closeSearcher [.system_shard1_replica2] Closing main searcher on request.
   [junit4]   2> 364809 T1782 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 364809 T1782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-006/.system_shard1_replica2/data [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-006/.system_shard1_replica2/data;done=false>>]
   [junit4]   2> 364810 T1782 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-006/.system_shard1_replica2/data
   [junit4]   2> 364810 T1782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-006/.system_shard1_replica2/data/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-006/.system_shard1_replica2/data/index;done=false>>]
   [junit4]   2> 364810 T1782 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001/tempDir-006/.system_shard1_replica2/data/index
   [junit4]   2> 364811 T1782 oasc.Overseer.close Overseer (id=93006250696704012-127.0.0.1:32789__iw%2Fnr-n_0000000004) closing
   [junit4]   2> 364812 T1945 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:32789__iw%2Fnr
   [junit4]   2> 366312 T1896 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 366313 T1940 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 366336 T1782 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/_iw/nr,null}
   [junit4]   2> 371326 T1782 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 371327 T1782 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:33501 33501
   [junit4]   2> 372165 T1783 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:33501 33501
   [junit4]   2> 372167 T1783 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		10	/solr/configs/conf1
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		2	/solr/configs/.system
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDynamicLoading -Dtests.method=testDistribSearch -Dtests.seed=61151B87BC111CF3 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es_CL -Dtests.timezone=Etc/GMT+4 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 50.2s J1 | TestDynamicLoading.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: {
   [junit4]    >   "responseHeader":{
   [junit4]    >     "status":404,
   [junit4]    >     "QTime":10},
   [junit4]    >   "error":{
   [junit4]    >     "msg":"no such blob or version available: test/1",
   [junit4]    >     "code":404}} expected:<org.apache.solr.core.BlobStoreTestRequestHandler> but was:<null>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61151B87BC111CF3:E0F3959FCB4E7CCF]:0)
   [junit4]    > 	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:101)
   [junit4]    > 	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 372199 T1782 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-61151B87BC111CF3-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {blobName=BlockTreeOrds(blocksize=128), _version_=FST50, id=PostingsFormat(name=LuceneFixedGap), version=BlockTreeOrds(blocksize=128), timestamp=FST50}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=es_CL, timezone=Etc/GMT+4
   [junit4]   2> NOTE: Linux 3.13.0-43-generic i386/Oracle Corporation 1.8.0_40-ea (32-bit)/cpus=8,threads=1,free=36665512,total=104976384
   [junit4]   2> NOTE: All tests run in this JVM: [QueryResultKeyTest, JsonLoaderTest, MoreLikeThisHandlerTest, TestRecoveryHdfs, TestManagedSchema, SimplePostToolTest, AutoCommitTest, TestJoin, TestElisionMultitermQuery, TestSchemaManager, TestIntervalFaceting, ChaosMonkeyNothingIsSafeTest, TestDynamicLoading]
   [junit4] Completed on J1 in 50.21s, 1 test, 1 failure <<< FAILURES!

[...truncated 1268 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:519: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:61: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:187: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:510: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1349: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:956: There were test failures: 453 suites, 1850 tests, 1 failure, 32 ignored (22 assumptions)

Total time: 91 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
[description-setter] Description set: Java: 32bit/jdk1.8.0_40-ea-b09 -client -XX:+UseSerialGC (asserts: true)
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any