You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/02/19 06:42:34 UTC

[JENKINS] Lucene-Solr-Tests-4.x-Java6 - Build # 2312 - Failure

Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java6/2312/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:63972/gt_bbp/sd/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:63972/gt_bbp/sd/collection1
	at __randomizedtesting.SeedInfo.seed([6E7AC50F23C07DFE:EF9C4B17549F1DC2]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.BaseDistributedSearchTestCase.add(BaseDistributedSearchTestCase.java:465)
	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:661)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:146)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:883)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:840)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:94)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 46 more




Build Log:
[...truncated 9947 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> 826002 T3073 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 826003 T3073 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /gt_bbp/sd
   [junit4]   2> 826009 T3073 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./solrtest-BasicDistributedZkTest-1392786712355
   [junit4]   2> 826011 T3073 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 826011 T3074 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 826112 T3073 oasc.ZkTestServer.run start zk server on port:63946
   [junit4]   2> 826114 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 826119 T3080 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c698e2 name:ZooKeeperConnection Watcher:127.0.0.1:63946 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 826119 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 826120 T3073 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 826138 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 826141 T3082 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bc3fae5 name:ZooKeeperConnection Watcher:127.0.0.1:63946/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 826141 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 826141 T3073 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 826152 T3073 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 826155 T3073 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 826158 T3073 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 826166 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 826167 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 826172 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 826173 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 826277 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 826279 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 826282 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 826282 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 826292 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 826293 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 826296 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 826297 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 826300 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 826301 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 826304 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 826305 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 826308 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 826308 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 826312 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 826312 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 826315 T3073 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 826316 T3073 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 826325 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 826327 T3084 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@645eb76a name:ZooKeeperConnection Watcher:127.0.0.1:63946/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 826328 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 826632 T3073 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 826638 T3073 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 826641 T3073 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:63958
   [junit4]   2> 826644 T3073 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 826645 T3073 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 826645 T3073 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681
   [junit4]   2> 826646 T3073 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681/'
   [junit4]   2> 826685 T3073 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681/solr.xml
   [junit4]   2> 826765 T3073 oasc.CoreContainer.<init> New CoreContainer 994294413
   [junit4]   2> 826765 T3073 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681/]
   [junit4]   2> 826767 T3073 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 826768 T3073 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 826768 T3073 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 826769 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 826769 T3073 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 826770 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 826770 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 826771 T3073 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 826771 T3073 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 826786 T3073 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 826786 T3073 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 826787 T3073 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 826788 T3073 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63946/solr
   [junit4]   2> 826788 T3073 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 826789 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 826792 T3095 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b4f8699 name:ZooKeeperConnection Watcher:127.0.0.1:63946 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 826792 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 826796 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 826807 T3097 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@662a7fd1 name:ZooKeeperConnection Watcher:127.0.0.1:63946/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 826807 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 826809 T3073 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 826821 T3073 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 826826 T3073 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 826829 T3073 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63958_gt_bbp%2Fsd
   [junit4]   2> 826830 T3073 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63958_gt_bbp%2Fsd
   [junit4]   2> 826841 T3073 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 826844 T3073 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 826857 T3073 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:63958_gt_bbp%2Fsd
   [junit4]   2> 826857 T3073 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 826860 T3073 oasc.Overseer.start Overseer (id=91277669981421572-127.0.0.1:63958_gt_bbp%2Fsd-n_0000000000) starting
   [junit4]   2> 826872 T3073 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 826884 T3099 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 826885 T3073 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 826885 T3099 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 826888 T3073 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 826890 T3073 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 826892 T3098 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 826898 T3100 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 826898 T3100 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 826899 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 826900 T3100 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 826901 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 826901 T3098 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63958/gt_bbp/sd",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63958_gt_bbp%2Fsd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 826902 T3098 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 826902 T3098 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 826904 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 826913 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 827901 T3100 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 827901 T3100 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681/collection1
   [junit4]   2> 827901 T3100 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 827902 T3100 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 827902 T3100 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 827903 T3100 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 827903 T3100 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681/collection1/'
   [junit4]   2> 827905 T3100 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681/collection1/lib/README' to classloader
   [junit4]   2> 827905 T3100 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681/collection1/lib/classes/' to classloader
   [junit4]   2> 827967 T3100 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 828034 T3100 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 828136 T3100 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 828146 T3100 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 828748 T3100 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 828749 T3100 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 828750 T3100 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 828765 T3100 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 828770 T3100 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 828783 T3100 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 828788 T3100 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 828793 T3100 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 828794 T3100 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 828794 T3100 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 828794 T3100 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 828795 T3100 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 828796 T3100 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 828796 T3100 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 828796 T3100 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392786712681/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/control/data/
   [junit4]   2> 828797 T3100 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@30d58ce2
   [junit4]   2> 828797 T3100 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/control/data
   [junit4]   2> 828798 T3100 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/control/data/index/
   [junit4]   2> 828798 T3100 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 828798 T3100 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/control/data/index
   [junit4]   2> 828799 T3100 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=87.16015625, floorSegmentMB=1.0849609375, forceMergeDeletesPctAllowed=5.145918506537807, segmentsPerTier=29.0, maxCFSSegmentSizeMB=2.0126953125, noCFSRatio=0.16981423433552312
   [junit4]   2> 828800 T3100 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41d4588a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@34df7ddd),segFN=segments_1,generation=1}
   [junit4]   2> 828800 T3100 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 828804 T3100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 828804 T3100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 828804 T3100 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 828805 T3100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 828805 T3100 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 828805 T3100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 828806 T3100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 828806 T3100 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 828807 T3100 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 828807 T3100 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 828808 T3100 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 828808 T3100 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 828809 T3100 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 828809 T3100 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 828810 T3100 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 828810 T3100 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 828820 T3100 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 828835 T3100 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 828835 T3100 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 828835 T3100 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=58.1455078125, floorSegmentMB=0.3583984375, forceMergeDeletesPctAllowed=10.710032891609576, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.29234990498634184
   [junit4]   2> 828836 T3100 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41d4588a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@34df7ddd),segFN=segments_1,generation=1}
   [junit4]   2> 828836 T3100 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 828837 T3100 oass.SolrIndexSearcher.<init> Opening Searcher@6b0123f1[collection1] main
   [junit4]   2> 828840 T3101 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6b0123f1[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 828842 T3100 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 828843 T3073 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1
   [junit4]   2> 828843 T3073 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 828843 T3104 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:63958/gt_bbp/sd collection:control_collection shard:shard1
   [junit4]   2> 828845 T3104 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 828848 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 828852 T3106 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1eaf30ec name:ZooKeeperConnection Watcher:127.0.0.1:63946/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 828852 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 828864 T3073 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 828891 T3104 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 828891 T3073 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 828937 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 828938 T3104 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 828938 T3104 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C845 name=collection1 org.apache.solr.core.SolrCore@345f4637 url=https://127.0.0.1:63958/gt_bbp/sd/collection1 node=127.0.0.1:63958_gt_bbp%2Fsd C845_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:63958/gt_bbp/sd, core=collection1, node_name=127.0.0.1:63958_gt_bbp%2Fsd}
   [junit4]   2> 828938 T3104 C845 P63958 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:63958/gt_bbp/sd/collection1/
   [junit4]   2> 828939 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 828939 T3104 C845 P63958 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 828939 T3104 C845 P63958 oasc.SyncStrategy.syncToMe https://127.0.0.1:63958/gt_bbp/sd/collection1/ has no replicas
   [junit4]   2> 828940 T3104 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:63958/gt_bbp/sd/collection1/ shard1
   [junit4]   2> 828940 T3104 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 828949 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 828960 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 828960 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 829062 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 829122 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 829175 T3073 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 829176 T3073 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 829180 T3073 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 829182 T3073 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:63962
   [junit4]   2> 829184 T3073 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 829185 T3073 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 829186 T3073 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236
   [junit4]   2> 829186 T3073 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236/'
   [junit4]   2> 829224 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 829224 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 829224 T3073 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236/solr.xml
   [junit4]   2> 829266 T3104 oasc.ZkController.register We are https://127.0.0.1:63958/gt_bbp/sd/collection1/ and leader is https://127.0.0.1:63958/gt_bbp/sd/collection1/
   [junit4]   2> 829266 T3104 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:63958/gt_bbp/sd
   [junit4]   2> 829266 T3104 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 829266 T3104 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 829267 T3104 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 829268 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 829268 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 829268 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 829268 T3104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 829270 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 829270 T3098 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63958/gt_bbp/sd",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63958_gt_bbp%2Fsd",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 829279 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 829302 T3073 oasc.CoreContainer.<init> New CoreContainer 871516469
   [junit4]   2> 829303 T3073 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236/]
   [junit4]   2> 829305 T3073 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 829305 T3073 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 829306 T3073 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 829306 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 829307 T3073 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 829307 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 829308 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 829308 T3073 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 829309 T3073 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 829323 T3073 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 829323 T3073 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 829324 T3073 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 829324 T3073 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63946/solr
   [junit4]   2> 829325 T3073 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 829326 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 829330 T3117 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@462ff407 name:ZooKeeperConnection Watcher:127.0.0.1:63946 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 829330 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 829333 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 829341 T3119 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fe1cb7d name:ZooKeeperConnection Watcher:127.0.0.1:63946/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 829341 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 829346 T3073 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 829387 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 829387 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 829387 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 830351 T3073 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63962_gt_bbp%2Fsd
   [junit4]   2> 830353 T3073 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63962_gt_bbp%2Fsd
   [junit4]   2> 830356 T3119 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 830356 T3097 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 830356 T3106 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 830376 T3120 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 830377 T3120 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 830378 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 830378 T3120 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 830378 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 830379 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 830380 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 830381 T3098 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63962/gt_bbp/sd",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63962_gt_bbp%2Fsd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 830381 T3098 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 830381 T3098 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 830385 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 830488 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 830488 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 830488 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 831380 T3120 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 831380 T3120 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236/collection1
   [junit4]   2> 831380 T3120 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 831381 T3120 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 831381 T3120 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 831382 T3120 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 831383 T3120 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236/collection1/'
   [junit4]   2> 831384 T3120 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236/collection1/lib/classes/' to classloader
   [junit4]   2> 831385 T3120 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236/collection1/lib/README' to classloader
   [junit4]   2> 831446 T3120 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 831520 T3120 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 831623 T3120 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 831633 T3120 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 832238 T3120 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 832238 T3120 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 832239 T3120 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 832251 T3120 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 832255 T3120 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 832269 T3120 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 832274 T3120 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 832278 T3120 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 832279 T3120 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 832280 T3120 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 832280 T3120 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 832281 T3120 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 832281 T3120 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 832281 T3120 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 832282 T3120 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392786715236/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty1/
   [junit4]   2> 832282 T3120 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@30d58ce2
   [junit4]   2> 832283 T3120 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty1
   [junit4]   2> 832283 T3120 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty1/index/
   [junit4]   2> 832284 T3120 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 832284 T3120 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty1/index
   [junit4]   2> 832284 T3120 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=87.16015625, floorSegmentMB=1.0849609375, forceMergeDeletesPctAllowed=5.145918506537807, segmentsPerTier=29.0, maxCFSSegmentSizeMB=2.0126953125, noCFSRatio=0.16981423433552312
   [junit4]   2> 832285 T3120 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2c669197 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@614080c9),segFN=segments_1,generation=1}
   [junit4]   2> 832286 T3120 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 832290 T3120 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 832290 T3120 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 832291 T3120 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 832291 T3120 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 832291 T3120 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 832292 T3120 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 832292 T3120 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 832292 T3120 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 832293 T3120 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 832293 T3120 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 832294 T3120 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 832294 T3120 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 832295 T3120 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 832295 T3120 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 832296 T3120 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 832297 T3120 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 832306 T3120 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 832311 T3120 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 832311 T3120 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 832312 T3120 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=58.1455078125, floorSegmentMB=0.3583984375, forceMergeDeletesPctAllowed=10.710032891609576, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.29234990498634184
   [junit4]   2> 832313 T3120 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2c669197 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@614080c9),segFN=segments_1,generation=1}
   [junit4]   2> 832313 T3120 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 832314 T3120 oass.SolrIndexSearcher.<init> Opening Searcher@44079576[collection1] main
   [junit4]   2> 832318 T3121 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@44079576[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 832320 T3120 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 832321 T3073 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1
   [junit4]   2> 832322 T3073 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 832321 T3124 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:63962/gt_bbp/sd collection:collection1 shard:shard1
   [junit4]   2> 832323 T3124 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 832341 T3124 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 832343 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 832344 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 832344 T3124 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 832344 T3124 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 832344 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C846 name=collection1 org.apache.solr.core.SolrCore@4f8109d6 url=https://127.0.0.1:63962/gt_bbp/sd/collection1 node=127.0.0.1:63962_gt_bbp%2Fsd C846_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:63962/gt_bbp/sd, core=collection1, node_name=127.0.0.1:63962_gt_bbp%2Fsd}
   [junit4]   2> 832345 T3124 C846 P63962 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:63962/gt_bbp/sd/collection1/
   [junit4]   2> 832345 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 832345 T3124 C846 P63962 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 832346 T3124 C846 P63962 oasc.SyncStrategy.syncToMe https://127.0.0.1:63962/gt_bbp/sd/collection1/ has no replicas
   [junit4]   2> 832346 T3124 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:63962/gt_bbp/sd/collection1/ shard1
   [junit4]   2> 832346 T3124 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 832348 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 832349 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832349 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832349 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832354 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 832356 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 832460 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832460 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832460 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832508 T3124 oasc.ZkController.register We are https://127.0.0.1:63962/gt_bbp/sd/collection1/ and leader is https://127.0.0.1:63962/gt_bbp/sd/collection1/
   [junit4]   2> 832508 T3124 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:63962/gt_bbp/sd
   [junit4]   2> 832508 T3124 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 832508 T3124 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 832508 T3124 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 832510 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 832510 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 832510 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 832510 T3124 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 832511 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 832512 T3098 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63962/gt_bbp/sd",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63962_gt_bbp%2Fsd",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 832521 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 832610 T3073 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 832611 T3073 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 832615 T3073 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 832616 T3073 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:63965
   [junit4]   2> 832619 T3073 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 832620 T3073 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 832620 T3073 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670
   [junit4]   2> 832621 T3073 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670/'
   [junit4]   2> 832623 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832624 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832623 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 832659 T3073 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670/solr.xml
   [junit4]   2> 832737 T3073 oasc.CoreContainer.<init> New CoreContainer 826088026
   [junit4]   2> 832738 T3073 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670/]
   [junit4]   2> 832740 T3073 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 832740 T3073 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 832741 T3073 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 832741 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 832742 T3073 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 832742 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 832743 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 832743 T3073 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 832744 T3073 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 832758 T3073 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 832759 T3073 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 832759 T3073 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 832760 T3073 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63946/solr
   [junit4]   2> 832760 T3073 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 832761 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 832765 T3135 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3c534daf name:ZooKeeperConnection Watcher:127.0.0.1:63946 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 832766 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 832769 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 832770 T3137 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ac291e1 name:ZooKeeperConnection Watcher:127.0.0.1:63946/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 832771 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 832783 T3073 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 833786 T3073 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63965_gt_bbp%2Fsd
   [junit4]   2> 833788 T3073 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63965_gt_bbp%2Fsd
   [junit4]   2> 833792 T3137 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 833792 T3097 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 833792 T3106 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 833792 T3119 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 833827 T3138 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 833827 T3138 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 833829 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 833829 T3138 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 833829 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 833829 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 833830 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 833831 T3098 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63965/gt_bbp/sd",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63965_gt_bbp%2Fsd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 833831 T3098 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 833831 T3098 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 833856 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 833960 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 833960 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 833960 T3137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 833960 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 834831 T3138 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 834831 T3138 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670/collection1
   [junit4]   2> 834831 T3138 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 834832 T3138 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 834833 T3138 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 834834 T3138 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 834834 T3138 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670/collection1/'
   [junit4]   2> 834835 T3138 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670/collection1/lib/classes/' to classloader
   [junit4]   2> 834836 T3138 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670/collection1/lib/README' to classloader
   [junit4]   2> 834898 T3138 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 834964 T3138 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 835067 T3138 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 835077 T3138 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 835679 T3138 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 835680 T3138 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 835681 T3138 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 835693 T3138 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 835697 T3138 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 835710 T3138 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 835715 T3138 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 835720 T3138 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 835721 T3138 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 835721 T3138 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 835721 T3138 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 835722 T3138 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 835722 T3138 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 835723 T3138 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 835723 T3138 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392786718670/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty2/
   [junit4]   2> 835723 T3138 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@30d58ce2
   [junit4]   2> 835724 T3138 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty2
   [junit4]   2> 835725 T3138 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty2/index/
   [junit4]   2> 835725 T3138 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 835725 T3138 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty2/index
   [junit4]   2> 835726 T3138 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=87.16015625, floorSegmentMB=1.0849609375, forceMergeDeletesPctAllowed=5.145918506537807, segmentsPerTier=29.0, maxCFSSegmentSizeMB=2.0126953125, noCFSRatio=0.16981423433552312
   [junit4]   2> 835727 T3138 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e6d1362 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@619db33e),segFN=segments_1,generation=1}
   [junit4]   2> 835727 T3138 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 835731 T3138 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 835732 T3138 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 835732 T3138 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 835732 T3138 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 835733 T3138 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 835733 T3138 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 835733 T3138 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 835734 T3138 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 835734 T3138 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 835735 T3138 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 835735 T3138 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 835736 T3138 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 835736 T3138 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 835737 T3138 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 835737 T3138 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 835738 T3138 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 835748 T3138 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 835753 T3138 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 835753 T3138 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 835754 T3138 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=58.1455078125, floorSegmentMB=0.3583984375, forceMergeDeletesPctAllowed=10.710032891609576, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.29234990498634184
   [junit4]   2> 835754 T3138 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e6d1362 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@619db33e),segFN=segments_1,generation=1}
   [junit4]   2> 835755 T3138 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 835755 T3138 oass.SolrIndexSearcher.<init> Opening Searcher@6a9a14bd[collection1] main
   [junit4]   2> 835759 T3139 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a9a14bd[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 835762 T3138 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 835763 T3073 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1
   [junit4]   2> 835763 T3073 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 835763 T3142 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:63965/gt_bbp/sd collection:collection1 shard:shard2
   [junit4]   2> 835765 T3142 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 835786 T3142 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 835795 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 835796 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 835796 T3142 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 835796 T3142 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 835796 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C847 name=collection1 org.apache.solr.core.SolrCore@57b95f9c url=https://127.0.0.1:63965/gt_bbp/sd/collection1 node=127.0.0.1:63965_gt_bbp%2Fsd C847_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:63965/gt_bbp/sd, core=collection1, node_name=127.0.0.1:63965_gt_bbp%2Fsd}
   [junit4]   2> 835796 T3142 C847 P63965 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:63965/gt_bbp/sd/collection1/
   [junit4]   2> 835797 T3142 C847 P63965 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 835797 T3142 C847 P63965 oasc.SyncStrategy.syncToMe https://127.0.0.1:63965/gt_bbp/sd/collection1/ has no replicas
   [junit4]   2> 835798 T3142 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:63965/gt_bbp/sd/collection1/ shard2
   [junit4]   2> 835798 T3142 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 835798 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 835802 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 835813 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 835813 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 835813 T3137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 835813 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 835818 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 835821 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 835924 T3137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 835924 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 835924 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 835924 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 835969 T3142 oasc.ZkController.register We are https://127.0.0.1:63965/gt_bbp/sd/collection1/ and leader is https://127.0.0.1:63965/gt_bbp/sd/collection1/
   [junit4]   2> 835969 T3142 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:63965/gt_bbp/sd
   [junit4]   2> 835969 T3142 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 835969 T3142 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 835970 T3142 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 835971 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 835971 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 835971 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 835971 T3142 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 835972 T3098 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 835973 T3098 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63965/gt_bbp/sd",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63965_gt_bbp%2Fsd",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 835983 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 836050 T3073 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 836051 T3073 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 836055 T3073 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 836057 T3073 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:63968
   [junit4]   2> 836060 T3073 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 836060 T3073 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 836061 T3073 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392786722111
   [junit4]   2> 836061 T3073 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392786722111/'
   [junit4]   2> 836086 T3137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 836086 T3119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 836086 T3097 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 836086 T3106 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 836100 T3073 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392786722111/solr.xml
   [junit4]   2> 836179 T3073 oasc.CoreContainer.<init> New CoreContainer 290713159
   [junit4]   2> 836179 T3073 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392786722111/]
   [junit4]   2> 836189 T3073 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 836190 T3073 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 836191 T3073 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 836191 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 836192 T3073 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 836192 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 836193 T3073 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 836193 T3073 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 836194 T3073 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 836208 T3073 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 836208 T3073 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 836209 T3073 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 836209 T3073 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63946/solr
   [junit4]   2> 836210 T3073 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 836211 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 836214 T3153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6295923f name:ZooKeeperConnection Watcher:127.0.0.1:63946 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 836214 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 836217 T3073 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 836224 T3155 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11a921b1 name:ZooKeeperConnection Watcher:127.0.0.1:63946/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 836224 T3073 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 836230 T3073 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 837234 T3073 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63968_gt_bbp%2Fsd
   [junit4]   2> 837236 T3073 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63968_gt_bbp%2Fsd
   [junit4]   2> 837240 T3106 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 837240 T3097 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 837240 T3155 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 837240 T3137 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 837240 T3119 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 837258 T3156 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 837258 T3156 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 837260 T3097 oasc.DistributedQueue$LatchChildWatcher.process LatchChil

[...truncated too long message...]

alue looking to close ./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty4/index;done=false>>]
   [junit4]   2> 958247 T3175 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.BasicDistributedZkTest-1392786712354/jetty4/index
   [junit4]   2> 958248 T3175 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection3/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 958248 T3175 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 958249 T3175 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection2/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 958249 T3175 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Lucene46: {foo_b=FST41, n_f1=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), cat=PostingsFormat(name=MockSep), foo_d=PostingsFormat(name=FSTPulsing41), foo_f=FST41, n_tl1=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), n_d1=PostingsFormat(name=MockSep), rnd_b=PostingsFormat(name=FSTPulsing41), intDefault=PostingsFormat(name=MockSep), n_td1=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), timestamp=PostingsFormat(name=MockSep), id=PostingsFormat(name=FSTPulsing41), range_facet_sl=FST41, range_facet_si=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), oddField_s=FST41, sequence_i=FST41, name=PostingsFormat(name=FSTPulsing41), foo_i=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), regex_dup_B_s=PostingsFormat(name=MockSep), multiDefault=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), n_tf1=PostingsFormat(name=MockSep), n_dt1=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), genre_s=FST41, author_t=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), range_facet_l=PostingsFormat(name=MockSep), n_ti1=PostingsFormat(name=FSTPulsing41), text=FST41, _version_=PostingsFormat(name=MockSep), val_i=PostingsFormat(name=FSTPulsing41), SubjectTerms_mfacet=PostingsFormat(name=MockSep), series_t=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), a_t=PostingsFormat(name=MockSep), n_tdt1=PostingsFormat(name=MockSep), regex_dup_A_s=FST41, price=FST41, other_tl1=PostingsFormat(name=MockSep), n_l1=PostingsFormat(name=MockSep), a_si=Pulsing41(freqCutoff=19 minBlockSize=33 maxBlockSize=113), inStock=FST41}, docValues:{}, sim=DefaultSimilarity, locale=fr_CH, timezone=America/Resolute
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=1,free=88708384,total=470024192
   [junit4]   2> NOTE: All tests run in this JVM: [ChaosMonkeySafeLeaderTest, TestOmitPositions, TestZkChroot, TestClassNameShortening, FullSolrCloudDistribCmdsTest, HighlighterTest, DefaultValueUpdateProcessorTest, TestJmxIntegration, TestStressUserVersions, TestFileDictionaryLookup, PreAnalyzedUpdateProcessorTest, TestBinaryField, UniqFieldsUpdateProcessorFactoryTest, StatsComponentTest, SpellCheckComponentTest, TestDistribDocBasedVersion, DocValuesMultiTest, TestArbitraryIndexDir, ChangedSchemaMergeTest, SpellingQueryConverterTest, ZkSolrClientTest, HdfsSyncSliceTest, TestNonDefinedSimilarityFactory, OutputWriterTest, NotRequiredUniqueKeyTest, TestCoreDiscovery, TermVectorComponentDistributedTest, XmlUpdateRequestHandlerTest, TestDFRSimilarityFactory, TestSolrQueryParserDefaultOperatorResource, SolrRequestParserTest, TestCopyFieldCollectionResource, TestDistributedGrouping, HdfsUnloadDistributedZkTest, DateFieldTest, JSONWriterTest, TestSolrXmlPersistence, DebugComponentTest, TestPhraseSuggestions, TestFieldTypeResource, TestAnalyzedSuggestions, SolrCmdDistributorTest, TestCloudManagedSchema, TestHashPartitioner, RAMDirectoryFactoryTest, TimeZoneUtilsTest, PrimUtilsTest, TestSolr4Spatial, TestUpdate, TestRandomFaceting, TestCodecSupport, DistributedQueryComponentCustomSortTest, TestInfoStreamLogging, TestImplicitCoreProperties, StandardRequestHandlerTest, TestCollationKeyRangeQueries, ShardRoutingCustomTest, SortByFunctionTest, TestDistributedSearch, TestSweetSpotSimilarityFactory, FileUtilsTest, TestRecovery, BadCopyFieldTest, TestReloadAndDeleteDocs, AssignTest, HardAutoCommitTest, MinimalSchemaTest, HdfsRecoveryZkTest, TestMultiCoreConfBootstrap, SimplePostToolTest, TestCollationFieldDocValues, TestRTGBase, DirectUpdateHandlerTest, CachingDirectoryFactoryTest, TestSolrCoreProperties, TestHighlightDedupGrouping, TestFoldingMultitermQuery, TestCollapseQParserPlugin, TestUtils, RequestHandlersTest, HdfsLockFactoryTest, TestBinaryResponseWriter, TestSolrXml, TestManagedSchemaFieldResource, DirectSolrSpellCheckerTest, TestLazyCores, BasicDistributedZkTest]
   [junit4] Completed on J1 in 132.56s, 1 test, 1 error <<< FAILURES!

[...truncated 679 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:905: There were test failures: 362 suites, 1563 tests, 1 error, 35 ignored (7 assumptions)

Total time: 70 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java6 - Build # 2315 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java6/2315/

2 tests failed.
REGRESSION:  org.apache.solr.cloud.UnloadDistributedZkTest.testDistribSearch

Error Message:
Captured an uncaught exception in thread: Thread[id=1469, name=testExecutor-167-thread-1, state=RUNNABLE, group=TGRP-UnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1469, name=testExecutor-167-thread-1, state=RUNNABLE, group=TGRP-UnloadDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:22053/s_ne/k
	at __randomizedtesting.SeedInfo.seed([AE4DD557F86E0BEE]:0)
	at org.apache.solr.cloud.UnloadDistributedZkTest$1.run(UnloadDistributedZkTest.java:393)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:679)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:22053/s_ne/k
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.cloud.UnloadDistributedZkTest$1.run(UnloadDistributedZkTest.java:391)
	... 3 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:146)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:883)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:840)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:94)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 5 more


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:22492/n_ok/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:22492/n_ok/collection1
	at __randomizedtesting.SeedInfo.seed([AE4DD557F86E0BEE:2FAB5B4F8F316BD2]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:168)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:146)
	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:663)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:146)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:883)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:840)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:94)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 47 more




Build Log:
[...truncated 9418 lines...]
   [junit4] Suite: org.apache.solr.cloud.UnloadDistributedZkTest
   [junit4]   2> 60760 T178 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 60761 T178 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /s_ne/k
   [junit4]   2> 60767 T178 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690
   [junit4]   2> 60768 T178 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 60769 T179 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 60870 T178 oasc.ZkTestServer.run start zk server on port:49808
   [junit4]   2> 60872 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 60902 T185 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e221f81 name:ZooKeeperConnection Watcher:127.0.0.1:49808 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 60902 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 60903 T178 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 60915 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 60918 T187 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dba4b54 name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 60918 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 60918 T178 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 60924 T178 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 60928 T178 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 60937 T178 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 60968 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 60969 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 60982 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 60983 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 61088 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 61089 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 61093 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 61094 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 61099 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 61100 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 61104 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 61105 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 61109 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 61110 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 61115 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 61115 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 61120 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 61121 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 61125 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 61126 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 61130 T178 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 61131 T178 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 61137 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 61142 T189 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e65b778 name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 61142 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 61496 T178 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 61503 T178 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 61505 T178 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:59310
   [junit4]   2> 61508 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 61509 T178 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 61510 T178 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.UnloadDistributedZkTest-controljetty-1393010584068
   [junit4]   2> 61510 T178 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.UnloadDistributedZkTest-controljetty-1393010584068/'
   [junit4]   2> 61561 T178 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-controljetty-1393010584068/solr.xml
   [junit4]   2> 61664 T178 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-controljetty-1393010584068
   [junit4]   2> 61665 T178 oasc.CoreContainer.<init> New CoreContainer 827130243
   [junit4]   2> 61666 T178 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.UnloadDistributedZkTest-controljetty-1393010584068/]
   [junit4]   2> 61668 T178 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 61669 T178 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 61669 T178 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 61670 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 61670 T178 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 61671 T178 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 61671 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 61672 T178 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 61672 T178 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 61689 T178 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 61690 T178 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 61691 T178 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 61691 T178 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49808/solr
   [junit4]   2> 61692 T178 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 61693 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 61696 T200 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f50388c name:ZooKeeperConnection Watcher:127.0.0.1:49808 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 61697 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 61711 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 61714 T202 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14a877c0 name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 61714 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 61721 T178 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 61726 T178 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 61738 T178 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 61741 T178 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59310_s_ne%2Fk
   [junit4]   2> 61743 T178 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59310_s_ne%2Fk
   [junit4]   2> 61748 T178 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 61751 T178 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 61764 T178 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:59310_s_ne%2Fk
   [junit4]   2> 61765 T178 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 61769 T178 oasc.Overseer.start Overseer (id=91292341613166596-127.0.0.1:59310_s_ne%2Fk-n_0000000000) starting
   [junit4]   2> 61779 T178 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 61790 T204 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 61791 T178 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 61791 T204 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 61794 T178 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 61797 T178 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 61801 T203 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 61804 T178 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-controljetty-1393010584068
   [junit4]   2> 61815 T178 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-controljetty-1393010584068/conf/
   [junit4]   2> 62033 T178 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 62034 T178 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0
   [junit4]   2> 62034 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 62041 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 62043 T207 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36b29562 name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 62043 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 62045 T178 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 62048 T178 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 62253 T178 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 62254 T178 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 62258 T178 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 62259 T178 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:12886
   [junit4]   2> 62261 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 62261 T178 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 62262 T178 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970
   [junit4]   2> 62262 T178 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970/'
   [junit4]   2> 62289 T178 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970/solr.xml
   [junit4]   2> 62347 T178 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970
   [junit4]   2> 62347 T178 oasc.CoreContainer.<init> New CoreContainer 338392849
   [junit4]   2> 62348 T178 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970/]
   [junit4]   2> 62349 T178 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 62350 T178 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 62350 T178 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 62350 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 62351 T178 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 62351 T178 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 62351 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 62352 T178 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 62352 T178 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 62363 T178 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 62363 T178 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 62364 T178 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 62364 T178 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49808/solr
   [junit4]   2> 62365 T178 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 62366 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 62368 T218 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@292f8a71 name:ZooKeeperConnection Watcher:127.0.0.1:49808 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 62369 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 62388 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 62390 T220 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@347f65f2 name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 62390 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 62402 T178 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 63406 T178 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:12886_s_ne%2Fk
   [junit4]   2> 63415 T178 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:12886_s_ne%2Fk
   [junit4]   2> 63419 T207 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 63419 T202 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 63419 T220 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 63430 T178 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970
   [junit4]   2> 63434 T178 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970/conf/
   [junit4]   2> 63784 T178 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 63785 T178 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0
   [junit4]   2> 63786 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 64144 T178 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 64145 T178 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 64151 T178 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 64152 T178 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:14150
   [junit4]   2> 64155 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 64156 T178 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 64157 T178 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711
   [junit4]   2> 64157 T178 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711/'
   [junit4]   2> 64203 T178 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711/solr.xml
   [junit4]   2> 64297 T178 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711
   [junit4]   2> 64298 T178 oasc.CoreContainer.<init> New CoreContainer 1934466975
   [junit4]   2> 64299 T178 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711/]
   [junit4]   2> 64301 T178 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 64301 T178 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 64302 T178 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 64302 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 64303 T178 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 64304 T178 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 64304 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 64305 T178 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 64305 T178 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 64320 T178 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 64321 T178 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 64321 T178 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 64322 T178 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49808/solr
   [junit4]   2> 64322 T178 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 64323 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 64325 T232 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1162bd90 name:ZooKeeperConnection Watcher:127.0.0.1:49808 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 64326 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 64329 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 64342 T234 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64fe7a67 name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 64342 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 64354 T178 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 65358 T178 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14150_s_ne%2Fk
   [junit4]   2> 65366 T178 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14150_s_ne%2Fk
   [junit4]   2> 65370 T207 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 65370 T234 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 65370 T220 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 65370 T202 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 65380 T178 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711
   [junit4]   2> 65710 T178 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711/conf/
   [junit4]   2> 65716 T178 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 65716 T178 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0
   [junit4]   2> 65717 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 66059 T178 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 66060 T178 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 66065 T178 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 66067 T178 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:22053
   [junit4]   2> 66070 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 66070 T178 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 66071 T178 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.UnloadDistributedZkTest-jetty3-1393010588642
   [junit4]   2> 66072 T178 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.UnloadDistributedZkTest-jetty3-1393010588642/'
   [junit4]   2> 66116 T178 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty3-1393010588642/solr.xml
   [junit4]   2> 66209 T178 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty3-1393010588642
   [junit4]   2> 66210 T178 oasc.CoreContainer.<init> New CoreContainer 1082105348
   [junit4]   2> 66211 T178 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.UnloadDistributedZkTest-jetty3-1393010588642/]
   [junit4]   2> 66213 T178 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 66213 T178 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 66214 T178 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 66214 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 66215 T178 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 66216 T178 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 66216 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 66217 T178 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 66217 T178 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 66234 T178 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 66234 T178 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 66235 T178 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 66236 T178 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49808/solr
   [junit4]   2> 66236 T178 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 66237 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 66245 T246 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6be3f34c name:ZooKeeperConnection Watcher:127.0.0.1:49808 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 66245 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 66249 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 66255 T248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@688fd95d name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 66256 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 66263 T178 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 67267 T178 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22053_s_ne%2Fk
   [junit4]   2> 67269 T178 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22053_s_ne%2Fk
   [junit4]   2> 67280 T220 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 67281 T248 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 67281 T202 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 67281 T207 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 67281 T234 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 67288 T178 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty3-1393010588642
   [junit4]   2> 67473 T178 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty3-1393010588642/conf/
   [junit4]   2> 67478 T178 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 67479 T178 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0
   [junit4]   2> 67479 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 67793 T178 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
   [junit4]   2> 67794 T178 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 67799 T178 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 67801 T178 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:28031
   [junit4]   2> 67804 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 67805 T178 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 67805 T178 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.UnloadDistributedZkTest-jetty4-1393010590404
   [junit4]   2> 67806 T178 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.UnloadDistributedZkTest-jetty4-1393010590404/'
   [junit4]   2> 67837 T178 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty4-1393010590404/solr.xml
   [junit4]   2> 67901 T178 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty4-1393010590404
   [junit4]   2> 67901 T178 oasc.CoreContainer.<init> New CoreContainer 1083467978
   [junit4]   2> 67902 T178 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.UnloadDistributedZkTest-jetty4-1393010590404/]
   [junit4]   2> 67903 T178 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 67904 T178 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 67904 T178 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 67904 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 67905 T178 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 67905 T178 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 67905 T178 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 67906 T178 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 67906 T178 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 67918 T178 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 67918 T178 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 67918 T178 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 67919 T178 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49808/solr
   [junit4]   2> 67919 T178 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 67920 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 67922 T260 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b5789a5 name:ZooKeeperConnection Watcher:127.0.0.1:49808 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 67923 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 67926 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 67927 T262 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d121b88 name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 67927 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 67941 T178 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 68946 T178 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28031_s_ne%2Fk
   [junit4]   2> 68948 T178 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28031_s_ne%2Fk
   [junit4]   2> 68952 T248 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 68952 T207 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 68953 T220 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 68953 T234 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 68952 T202 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 68953 T262 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 68962 T178 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty4-1393010590404
   [junit4]   2> 69167 T178 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty4-1393010590404/conf/
   [junit4]   2> 69176 T178 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 69177 T178 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0
   [junit4]   2> 69177 T178 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 69298 T210 oasc.ZkController.publish publishing core=unloadcollection1 state=down
   [junit4]   2> 69300 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 69300 T210 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 69303 T203 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 69304 T203 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:12886/s_ne/k",
   [junit4]   2> 	  "core":"unloadcollection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:12886_s_ne%2Fk",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"unloadcollection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 69304 T203 oasc.Overseer$ClusterStateUpdater.createCollection Create collection unloadcollection with shards [shard1]
   [junit4]   2> 69304 T203 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 69314 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 69321 T248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 69321 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 69321 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 69321 T262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 69321 T220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 69321 T207 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 70302 T210 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for unloadcollection1
   [junit4]   2> 70311 T210 oasc.CoreContainer.create Creating SolrCore 'unloadcollection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970/unloadcollection1
   [junit4]   2> 70311 T210 oasc.ZkController.createCollectionZkNode Check for collection zkNode:unloadcollection
   [junit4]   2> 70313 T210 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:unloadcollection
   [junit4]   2> 70313 T210 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 70315 T210 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 70315 T210 oascc.SolrZkClient.makePath makePath: /collections/unloadcollection
   [junit4]   2> 70338 T210 oascc.ZkStateReader.readConfigName Load collection config from:/collections/unloadcollection
   [junit4]   2> 70340 T210 oascc.ZkStateReader.readConfigName path=/collections/unloadcollection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 70340 T210 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970/unloadcollection1/'
   [junit4]   2> 70443 T210 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 70522 T210 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 70624 T210 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 70640 T210 oass.IndexSchema.readSchema [unloadcollection1] Schema name=test
   [junit4]   2> 71326 T210 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 71327 T210 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 71327 T210 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 71337 T210 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 71341 T210 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 71362 T210 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 71367 T210 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 71371 T210 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 71372 T210 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 71372 T210 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 71373 T210 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 71374 T210 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 71374 T210 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 71375 T210 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 71375 T210 oasc.SolrCore.<init> [unloadcollection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty1-1393010584970/unloadcollection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010592105unloadcollection1_1n/
   [junit4]   2> 71375 T210 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1414627a
   [junit4]   2> 71376 T210 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010592105unloadcollection1_1n
   [junit4]   2> 71377 T210 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010592105unloadcollection1_1n/index/
   [junit4]   2> 71377 T210 oasc.SolrCore.initIndex WARN [unloadcollection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010592105unloadcollection1_1n/index' doesn't exist. Creating new index...
   [junit4]   2> 71379 T210 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010592105unloadcollection1_1n/index
   [junit4]   2> 71379 T210 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=50.5322265625, floorSegmentMB=2.1982421875, forceMergeDeletesPctAllowed=7.941642981308324, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8310669142710386
   [junit4]   2> 71381 T210 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@376433e4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4df194d9; maxCacheMB=0.15234375 maxMergeSizeMB=0.93359375)),segFN=segments_1,generation=1}
   [junit4]   2> 71381 T210 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 71385 T210 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 71386 T210 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 71386 T210 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 71386 T210 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 71387 T210 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 71387 T210 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 71388 T210 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 71388 T210 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 71388 T210 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 71389 T210 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 71390 T210 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 71390 T210 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 71390 T210 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 71391 T210 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 71392 T210 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 71392 T210 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 71399 T210 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 71403 T210 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 71404 T210 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 71404 T210 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=1.4384765625, floorSegmentMB=0.7900390625, forceMergeDeletesPctAllowed=25.352681800798376, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 71405 T210 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@376433e4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4df194d9; maxCacheMB=0.15234375 maxMergeSizeMB=0.93359375)),segFN=segments_1,generation=1}
   [junit4]   2> 71406 T210 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 71406 T210 oass.SolrIndexSearcher.<init> Opening Searcher@a0b96a5[unloadcollection1] main
   [junit4]   2> 71409 T265 oasc.SolrCore.registerSearcher [unloadcollection1] Registered new searcher Searcher@a0b96a5[unloadcollection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 71411 T210 oasc.CoreContainer.registerCore registering core: unloadcollection1
   [junit4]   2> 71412 T210 oasc.ZkController.register Register replica - core:unloadcollection1 address:https://127.0.0.1:12886/s_ne/k collection:unloadcollection shard:shard1
   [junit4]   2> 71413 T210 oascc.SolrZkClient.makePath makePath: /collections/unloadcollection/leader_elect/shard1/election
   [junit4]   2> 71433 T210 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 71450 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71451 T210 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 71452 T210 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C27 name=unloadcollection1 org.apache.solr.core.SolrCore@45715c20 url=https://127.0.0.1:12886/s_ne/k/unloadcollection1 node=127.0.0.1:12886_s_ne%2Fk C27_STATE=coll:unloadcollection core:unloadcollection1 props:{state=down, base_url=https://127.0.0.1:12886/s_ne/k, core=unloadcollection1, node_name=127.0.0.1:12886_s_ne%2Fk}
   [junit4]   2> 71452 T210 C27 P12886 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:12886/s_ne/k/unloadcollection1/
   [junit4]   2> 71452 T203 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71453 T210 C27 P12886 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 71453 T210 C27 P12886 oasc.SyncStrategy.syncToMe https://127.0.0.1:12886/s_ne/k/unloadcollection1/ has no replicas
   [junit4]   2> 71454 T210 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:12886/s_ne/k/unloadcollection1/ shard1
   [junit4]   2> 71454 T210 oascc.SolrZkClient.makePath makePath: /collections/unloadcollection/leaders/shard1
   [junit4]   2> 71463 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71470 T248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71470 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71470 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71470 T262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71470 T207 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71470 T220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71488 T203 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71502 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71613 T248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71613 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71613 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71613 T207 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71613 T262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71613 T220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71640 T210 oasc.ZkController.register We are https://127.0.0.1:12886/s_ne/k/unloadcollection1/ and leader is https://127.0.0.1:12886/s_ne/k/unloadcollection1/
   [junit4]   2> 71640 T210 oasc.ZkController.register No LogReplay needed for core=unloadcollection1 baseURL=https://127.0.0.1:12886/s_ne/k
   [junit4]   2> 71641 T210 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 71641 T210 oasc.ZkController.publish publishing core=unloadcollection1 state=active
   [junit4]   2> 71653 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71653 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71653 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71654 T210 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71655 T203 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71655 T210 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={numShards=1&name=unloadcollection1&action=CREATE&collection=unloadcollection&dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010592105unloadcollection1_1n&wt=javabin&version=2} status=0 QTime=2359 
   [junit4]   2> 71656 T203 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:12886/s_ne/k",
   [junit4]   2> 	  "core":"unloadcollection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:12886_s_ne%2Fk",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"unloadcollection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 71660 T178 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 71672 T268 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d076683 name:ZooKeeperConnection Watcher:127.0.0.1:49808/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 71673 T178 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 71674 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71675 T178 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 71678 T178 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71781 T248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71781 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71781 T268 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71781 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71781 T262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71781 T207 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71781 T220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71791 T224 oasc.ZkController.publish publishing core=unloadcollection2 state=down
   [junit4]   2> 71791 T224 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 71793 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71793 T224 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 71793 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71794 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71795 T203 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 71796 T203 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:14150/s_ne/k",
   [junit4]   2> 	  "core":"unloadcollection2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:14150_s_ne%2Fk",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"unloadcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 71796 T203 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 71796 T203 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 71808 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 71911 T207 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71911 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71911 T220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71911 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71911 T268 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71911 T262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 71911 T248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 72795 T224 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for unloadcollection2
   [junit4]   2> 72798 T224 oasc.CoreContainer.create Creating SolrCore 'unloadcollection2' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711/unloadcollection2
   [junit4]   2> 72798 T224 oasc.ZkController.createCollectionZkNode Check for collection zkNode:unloadcollection
   [junit4]   2> 72800 T224 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 72800 T224 oascc.ZkStateReader.readConfigName Load collection config from:/collections/unloadcollection
   [junit4]   2> 72802 T224 oascc.ZkStateReader.readConfigName path=/collections/unloadcollection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 72802 T224 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711/unloadcollection2/'
   [junit4]   2> 72865 T224 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 72941 T224 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 73043 T224 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 73058 T224 oass.IndexSchema.readSchema [unloadcollection2] Schema name=test
   [junit4]   2> 73757 T224 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 73758 T224 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 73759 T224 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 73770 T224 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 73777 T224 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 73806 T224 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 73812 T224 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 73819 T224 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 73820 T224 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 73821 T224 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 73821 T224 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 73823 T224 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 73823 T224 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 73824 T224 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 73825 T224 oasc.SolrCore.<init> [unloadcollection2] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./org.apache.solr.cloud.UnloadDistributedZkTest-jetty2-1393010586711/unloadcollection2/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010594604unloadcollection1_2n/
   [junit4]   2> 73825 T224 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1414627a
   [junit4]   2> 73827 T224 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010594604unloadcollection1_2n
   [junit4]   2> 73827 T224 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010594604unloadcollection1_2n/index/
   [junit4]   2> 73828 T224 oasc.SolrCore.initIndex WARN [unloadcollection2] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010594604unloadcollection1_2n/index' doesn't exist. Creating new index...
   [junit4]   2> 73828 T224 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010594604unloadcollection1_2n/index
   [junit4]   2> 73829 T224 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=50.5322265625, floorSegmentMB=2.1982421875, forceMergeDeletesPctAllowed=7.941642981308324, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8310669142710386
   [junit4]   2> 73831 T224 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@3caf7a1f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2843d653; maxCacheMB=0.15234375 maxMergeSizeMB=0.93359375)),segFN=segments_1,generation=1}
   [junit4]   2> 73831 T224 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 73837 T224 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 73838 T224 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 73838 T224 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 73839 T224 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 73839 T224 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 73840 T224 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 73841 T224 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 73841 T224 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 73842 T224 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 73843 T224 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 73843 T224 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 73844 T224 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 73844 T224 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 73845 T224 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 73846 T224 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 73847 T224 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 73849 T224 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 73856 T224 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 73856 T224 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 73857 T224 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=1.4384765625, floorSegmentMB=0.7900390625, forceMergeDeletesPctAllowed=25.352681800798376, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 73858 T224 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@3caf7a1f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2843d653; maxCacheMB=0.15234375 maxMergeSizeMB=0.93359375)),segFN=segments_1,generation=1}
   [junit4]   2> 73859 T224 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 73860 T224 oass.SolrIndexSearcher.<init> Opening Searcher@2ca7563d[unloadcollection2] main
   [junit4]   2> 73864 T270 oasc.SolrCore.registerSearcher [unloadcollection2] Registered new searcher Searcher@2ca7563d[unloadcollection2] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 73867 T224 oasc.CoreContainer.registerCore registering core: unloadcollection2
   [junit4]   2> 73867 T224 oasc.ZkController.register Register replica - core:unloadcollection2 address:https://127.0.0.1:14150/s_ne/k collection:unloadcollection shard:shard1
   [junit4]   2> 73887 T224 oasc.ZkController.register We are https://127.0.0.1:14150/s_ne/k/unloadcollection2/ and leader is https://127.0.0.1:12886/s_ne/k/unloadcollection1/
   [junit4]   2> 73887 T224 oasc.ZkController.register No LogReplay needed for core=unloadcollection2 baseURL=https://127.0.0.1:14150/s_ne/k
   [junit4]   2> 73888 T224 oasc.ZkController.checkRecovery Core needs to recover:unloadcollection2
   [junit4]   2> 73888 T224 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C28 name=unloadcollection2 org.apache.solr.core.SolrCore@230a4761 url=https://127.0.0.1:14150/s_ne/k/unloadcollection2 node=127.0.0.1:14150_s_ne%2Fk C28_STATE=coll:unloadcollection core:unloadcollection2 props:{state=down, base_url=https://127.0.0.1:14150/s_ne/k, core=unloadcollection2, node_name=127.0.0.1:14150_s_ne%2Fk}
   [junit4]   2> 73889 T272 C28 P14150 oasc.RecoveryStrategy.run Starting recovery process.  core=unloadcollection2 recoveringAfterStartup=true
   [junit4]   2> 73889 T272 C28 P14150 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 73889 T224 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73890 T272 C28 P14150 oasc.ZkController.publish publishing core=unloadcollection2 state=recovering
   [junit4]   2> 73891 T272 C28 P14150 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 73891 T224 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={name=unloadcollection2&action=CREATE&collection=unloadcollection&dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-UnloadDistributedZkTest-1393010583690/1393010594604unloadcollection1_2n&wt=javabin&version=2} status=0 QTime=2102 
   [junit4]   2> 73892 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73893 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73893 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73893 T178 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73894 T178 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: unloadcollection failOnTimeout:true timeout (sec):330
   [junit4]   2> 73895 T203 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73895 T178 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 73895 T203 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:14150/s_ne/k",
   [junit4]   2> 	  "core":"unloadcollection2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:14150_s_ne%2Fk",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"unloadcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 73900 T202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 73909 T248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 73909 T202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 73909 T220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 73909 T268 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 73909 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 73909 T207 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 73909 T262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 74003 T211 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true
   [junit4]   2> 74004 T211 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: core_

[...truncated too long message...]

ose ./org.apache.solr.cloud.BasicDistributedZkTest-1393013067245/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.BasicDistributedZkTest-1393013067245/jetty4/index;done=false>>]
   [junit4]   2> 2876507 T13093 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.BasicDistributedZkTest-1393013067245/jetty4/index
   [junit4]   2> 2876507 T13093 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.BasicDistributedZkTest-1393013067245/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.BasicDistributedZkTest-1393013067245/jetty4;done=false>>]
   [junit4]   2> 2876507 T13093 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.BasicDistributedZkTest-1393013067245/jetty4
   [junit4]   2> 2876508 T13093 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection2/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2876508 T13093 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection3/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2876509 T13093 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2876509 T13093 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Lucene46: {foo_b=FSTOrd41, n_f1=PostingsFormat(name=TestBloomFilteredLucene41Postings), cat=PostingsFormat(name=MockSep), foo_d=PostingsFormat(name=FSTOrdPulsing41), foo_f=FSTOrd41, n_tl1=PostingsFormat(name=TestBloomFilteredLucene41Postings), n_d1=PostingsFormat(name=MockSep), rnd_b=PostingsFormat(name=FSTOrdPulsing41), intDefault=PostingsFormat(name=TestBloomFilteredLucene41Postings), n_td1=PostingsFormat(name=TestBloomFilteredLucene41Postings), timestamp=PostingsFormat(name=MockSep), id=PostingsFormat(name=FSTOrdPulsing41), range_facet_sl=PostingsFormat(name=FSTOrdPulsing41), range_facet_si=PostingsFormat(name=MockSep), oddField_s=PostingsFormat(name=FSTOrdPulsing41), sequence_i=PostingsFormat(name=FSTOrdPulsing41), name=PostingsFormat(name=FSTOrdPulsing41), foo_i=PostingsFormat(name=TestBloomFilteredLucene41Postings), regex_dup_B_s=PostingsFormat(name=MockSep), multiDefault=PostingsFormat(name=MockSep), n_tf1=PostingsFormat(name=MockSep), n_dt1=PostingsFormat(name=TestBloomFilteredLucene41Postings), genre_s=PostingsFormat(name=FSTOrdPulsing41), author_t=PostingsFormat(name=TestBloomFilteredLucene41Postings), range_facet_l=PostingsFormat(name=MockSep), n_ti1=PostingsFormat(name=FSTOrdPulsing41), text=FSTOrd41, _version_=PostingsFormat(name=TestBloomFilteredLucene41Postings), val_i=PostingsFormat(name=FSTOrdPulsing41), SubjectTerms_mfacet=PostingsFormat(name=TestBloomFilteredLucene41Postings), series_t=PostingsFormat(name=TestBloomFilteredLucene41Postings), a_t=PostingsFormat(name=MockSep), n_tdt1=PostingsFormat(name=TestBloomFilteredLucene41Postings), regex_dup_A_s=FSTOrd41, price=FSTOrd41, other_tl1=PostingsFormat(name=TestBloomFilteredLucene41Postings), n_l1=PostingsFormat(name=MockSep), a_si=PostingsFormat(name=TestBloomFilteredLucene41Postings), inStock=FSTOrd41}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=es_HN, timezone=Etc/GMT-9
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=1,free=226237712,total=496828416
   [junit4]   2> NOTE: All tests run in this JVM: [RemoteQueryErrorTest, SpellPossibilityIteratorTest, SolrIndexSplitterTest, DirectUpdateHandlerOptimizeTest, UnloadDistributedZkTest, TestWriterPerf, LegacyHTMLStripCharFilterTest, ReturnFieldsTest, SystemInfoHandlerTest, ScriptEngineTest, TestRandomDVFaceting, SchemaVersionSpecificBehaviorTest, TestDefaultSearchFieldResource, TermVectorComponentTest, PeerSyncTest, ShowFileRequestHandlerTest, BinaryUpdateRequestHandlerTest, DocValuesMissingTest, CoreAdminHandlerTest, FastVectorHighlighterTest, SoftAutoCommitTest, AddSchemaFieldsUpdateProcessorFactoryTest, OverseerTest, LeaderElectionTest, TestQueryUtils, CSVRequestHandlerTest, AliasIntegrationTest, SliceStateUpdateTest, TestFunctionQuery, BasicFunctionalityTest, TestReplicationHandler, InfoHandlerTest, TestRemoteStreaming, SpatialFilterTest, SyncSliceTest, HdfsDirectoryTest, TestPerFieldSimilarity, FileBasedSpellCheckerTest, BadComponentTest, EnumFieldTest, TestSuggestSpellingConverter, ShardSplitTest, CoreMergeIndexesAdminHandlerTest, DeleteReplicaTest, TestJmxMonitoredMap, TestBlendedInfixSuggestions, AnalysisErrorHandlingTest, TestTrie, ZkCLITest, TestRealTimeGet, TestCharFilters, DistributedDebugComponentTest, TestDocumentBuilder, TestPseudoReturnFields, DOMUtilTest, TestSearchPerf, ClusterStateTest, TestMergePolicyConfig, JsonLoaderTest, CoreAdminCreateDiscoverTest, QueryEqualityTest, BasicDistributedZkTest]
   [junit4] Completed on J0 in 333.23s, 1 test, 1 error <<< FAILURES!

[...truncated 98 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:905: There were test failures: 362 suites, 1563 tests, 2 errors, 35 ignored (7 assumptions)

Total time: 84 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java6 - Build # 2314 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java6/2314/

3 tests failed.
REGRESSION:  org.apache.solr.cloud.OverseerTest.testOverseerFailure

Error Message:
KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1

Stack Trace:
org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at __randomizedtesting.SeedInfo.seed([AAD8573C542CBAAB:AED0D8CF4689558A]:0)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:429)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:426)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:383)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:370)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:35837/crsy/collection1, https://127.0.0.1:35868/crsy/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:35837/crsy/collection1, https://127.0.0.1:35868/crsy/collection1]
	at __randomizedtesting.SeedInfo.seed([AAD8573C542CBAAB:2B3ED9242373DA97]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:387)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:635)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.queryServer(AbstractFullDistribZkTestBase.java:1427)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:524)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:296)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:113)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: no servers hosting shard: 
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:302)
	... 50 more


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:32011/nq_yr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:32011/nq_yr/collection1
	at __randomizedtesting.SeedInfo.seed([AAD8573C542CBAAB:2B3ED9242373DA97]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.BaseDistributedSearchTestCase.add(BaseDistributedSearchTestCase.java:465)
	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:661)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:146)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:883)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:840)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:94)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 46 more




Build Log:
[...truncated 9866 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> 1169959 T8705 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 1169959 T8705 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /nq_yr/
   [junit4]   2> 1169964 T8705 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./solrtest-BasicDistributedZkTest-1392905311970
   [junit4]   2> 1169966 T8705 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1169966 T8706 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1170068 T8705 oasc.ZkTestServer.run start zk server on port:31986
   [junit4]   2> 1170069 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1170099 T8712 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@161957fb name:ZooKeeperConnection Watcher:127.0.0.1:31986 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1170099 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1170100 T8705 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1170111 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1170113 T8714 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5497d1bd name:ZooKeeperConnection Watcher:127.0.0.1:31986/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1170113 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1170114 T8705 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1170117 T8705 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1170127 T8705 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1170129 T8705 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1170137 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1170138 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1170142 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1170143 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1170249 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1170250 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1170253 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1170253 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1170256 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1170257 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1170264 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1170265 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1170268 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1170269 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1170272 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1170272 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1170275 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1170276 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1170283 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1170284 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1170287 T8705 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1170287 T8705 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1170298 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1170300 T8716 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e61ca name:ZooKeeperConnection Watcher:127.0.0.1:31986/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1170300 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1170670 T8705 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1170675 T8705 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 1170682 T8705 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:31990
   [junit4]   2> 1170685 T8705 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1170686 T8705 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1170687 T8705 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316
   [junit4]   2> 1170687 T8705 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316/'
   [junit4]   2> 1170731 T8705 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316/solr.xml
   [junit4]   2> 1170827 T8705 oasc.CoreContainer.<init> New CoreContainer 1209014296
   [junit4]   2> 1170828 T8705 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316/]
   [junit4]   2> 1170829 T8705 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1170830 T8705 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1170830 T8705 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1170831 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1170831 T8705 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1170832 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1170832 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1170833 T8705 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1170833 T8705 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1170848 T8705 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1170849 T8705 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1170849 T8705 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1170850 T8705 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31986/solr
   [junit4]   2> 1170850 T8705 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1170851 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1170854 T8727 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3c61ee76 name:ZooKeeperConnection Watcher:127.0.0.1:31986 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1170854 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1170898 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1170900 T8729 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@175db5e9 name:ZooKeeperConnection Watcher:127.0.0.1:31986/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1170901 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1170903 T8705 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1170917 T8705 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1170921 T8705 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1170923 T8705 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31990_nq_yr
   [junit4]   2> 1170931 T8705 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31990_nq_yr
   [junit4]   2> 1170934 T8705 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1170936 T8705 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1170951 T8705 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:31990_nq_yr
   [junit4]   2> 1170951 T8705 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1170954 T8705 oasc.Overseer.start Overseer (id=91285442525790212-127.0.0.1:31990_nq_yr-n_0000000000) starting
   [junit4]   2> 1170959 T8705 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1170973 T8731 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1170974 T8705 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1170974 T8731 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 1170976 T8705 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1170983 T8705 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1170986 T8730 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1170992 T8732 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1170993 T8732 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1170994 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1170994 T8732 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1170995 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1170996 T8730 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:31990/nq_yr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31990_nq_yr",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1170996 T8730 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1170997 T8730 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1171000 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1171001 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1171995 T8732 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1171995 T8732 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316/collection1
   [junit4]   2> 1171995 T8732 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1171996 T8732 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1171996 T8732 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1171997 T8732 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1171998 T8732 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316/collection1/'
   [junit4]   2> 1172000 T8732 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316/collection1/lib/classes/' to classloader
   [junit4]   2> 1172001 T8732 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316/collection1/lib/README' to classloader
   [junit4]   2> 1172061 T8732 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 1172128 T8732 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1172230 T8732 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1172240 T8732 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1172824 T8732 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1172824 T8732 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1172825 T8732 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1172833 T8732 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1172836 T8732 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1172847 T8732 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1172850 T8732 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1172853 T8732 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1172854 T8732 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1172855 T8732 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1172855 T8732 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1172856 T8732 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1172856 T8732 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1172856 T8732 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1172856 T8732 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392905312316/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/control/data/
   [junit4]   2> 1172857 T8732 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2fd0bf38
   [junit4]   2> 1172857 T8732 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/control/data
   [junit4]   2> 1172857 T8732 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/control/data/index/
   [junit4]   2> 1172858 T8732 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1172858 T8732 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/control/data/index
   [junit4]   2> 1172858 T8732 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.190651796096072]
   [junit4]   2> 1172859 T8732 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53c1d63c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f6c79c4),segFN=segments_1,generation=1}
   [junit4]   2> 1172859 T8732 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1172861 T8732 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1172861 T8732 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1172862 T8732 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1172862 T8732 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1172862 T8732 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1172862 T8732 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1172862 T8732 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1172863 T8732 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1172863 T8732 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1172863 T8732 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1172864 T8732 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1172864 T8732 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1172864 T8732 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1172864 T8732 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1172865 T8732 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1172865 T8732 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1172871 T8732 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1172875 T8732 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1172875 T8732 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1172876 T8732 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=53109793, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1172876 T8732 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53c1d63c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f6c79c4),segFN=segments_1,generation=1}
   [junit4]   2> 1172877 T8732 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1172877 T8732 oass.SolrIndexSearcher.<init> Opening Searcher@63d80a65[collection1] main
   [junit4]   2> 1172879 T8733 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@63d80a65[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1172879 T8732 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1172880 T8705 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1
   [junit4]   2> 1172881 T8705 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1172880 T8736 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:31990/nq_yr collection:control_collection shard:shard1
   [junit4]   2> 1172882 T8736 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1172885 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1172893 T8738 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56d463e6 name:ZooKeeperConnection Watcher:127.0.0.1:31986/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1172893 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1172901 T8705 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1172904 T8736 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1172904 T8705 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1172915 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1172915 T8736 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1172916 T8736 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C983 name=collection1 org.apache.solr.core.SolrCore@38599d7 url=https://127.0.0.1:31990/nq_yr/collection1 node=127.0.0.1:31990_nq_yr C983_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:31990/nq_yr, core=collection1, node_name=127.0.0.1:31990_nq_yr}
   [junit4]   2> 1172916 T8736 C983 P31990 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:31990/nq_yr/collection1/
   [junit4]   2> 1172916 T8736 C983 P31990 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1172916 T8736 C983 P31990 oasc.SyncStrategy.syncToMe https://127.0.0.1:31990/nq_yr/collection1/ has no replicas
   [junit4]   2> 1172916 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1172916 T8736 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:31990/nq_yr/collection1/ shard1
   [junit4]   2> 1172917 T8736 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1172920 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1172921 T8738 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1172921 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1172934 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1172944 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1173047 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1173047 T8738 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1173086 T8736 oasc.ZkController.register We are https://127.0.0.1:31990/nq_yr/collection1/ and leader is https://127.0.0.1:31990/nq_yr/collection1/
   [junit4]   2> 1173086 T8736 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:31990/nq_yr
   [junit4]   2> 1173086 T8736 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1173086 T8736 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1173086 T8736 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1173087 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1173087 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1173088 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1173088 T8736 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1173089 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1173089 T8730 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:31990/nq_yr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31990_nq_yr",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1173098 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1173190 T8705 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1173192 T8705 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1173196 T8705 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 1173197 T8705 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:31999
   [junit4]   2> 1173200 T8705 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1173200 T8705 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1173201 T8705 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909
   [junit4]   2> 1173201 T8705 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909/'
   [junit4]   2> 1173201 T8738 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1173201 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1173238 T8705 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909/solr.xml
   [junit4]   2> 1173316 T8705 oasc.CoreContainer.<init> New CoreContainer 525883358
   [junit4]   2> 1173316 T8705 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909/]
   [junit4]   2> 1173318 T8705 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1173319 T8705 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1173319 T8705 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1173320 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1173320 T8705 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1173321 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1173321 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1173321 T8705 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1173322 T8705 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1173336 T8705 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1173337 T8705 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1173337 T8705 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1173338 T8705 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31986/solr
   [junit4]   2> 1173338 T8705 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1173339 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1173341 T8749 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6bdd6c2b name:ZooKeeperConnection Watcher:127.0.0.1:31986 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1173342 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1173345 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1173353 T8751 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33c18f96 name:ZooKeeperConnection Watcher:127.0.0.1:31986/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1173354 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1173363 T8705 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1174366 T8705 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31999_nq_yr
   [junit4]   2> 1174377 T8705 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31999_nq_yr
   [junit4]   2> 1174387 T8729 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1174387 T8751 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1174387 T8738 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1174404 T8752 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1174405 T8752 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1174408 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1174408 T8752 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1174408 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1174409 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1174409 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1174410 T8730 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:31999/nq_yr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31999_nq_yr",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1174410 T8730 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1174410 T8730 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1174424 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1174527 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1174527 T8751 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1174527 T8738 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1175410 T8752 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1175410 T8752 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909/collection1
   [junit4]   2> 1175410 T8752 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1175411 T8752 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1175411 T8752 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1175412 T8752 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1175413 T8752 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909/collection1/'
   [junit4]   2> 1175414 T8752 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909/collection1/lib/classes/' to classloader
   [junit4]   2> 1175414 T8752 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909/collection1/lib/README' to classloader
   [junit4]   2> 1175475 T8752 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 1175544 T8752 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1175646 T8752 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1175656 T8752 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1176254 T8752 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1176255 T8752 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1176255 T8752 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1176270 T8752 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1176274 T8752 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1176288 T8752 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1176293 T8752 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1176297 T8752 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1176298 T8752 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1176299 T8752 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1176299 T8752 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1176301 T8752 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1176301 T8752 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1176301 T8752 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1176302 T8752 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392905314909/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty1/
   [junit4]   2> 1176302 T8752 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2fd0bf38
   [junit4]   2> 1176303 T8752 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty1
   [junit4]   2> 1176303 T8752 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty1/index/
   [junit4]   2> 1176303 T8752 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1176304 T8752 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty1/index
   [junit4]   2> 1176304 T8752 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.190651796096072]
   [junit4]   2> 1176305 T8752 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7e8846ae lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@76fe9833),segFN=segments_1,generation=1}
   [junit4]   2> 1176305 T8752 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1176308 T8752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1176309 T8752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1176309 T8752 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1176309 T8752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1176310 T8752 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1176310 T8752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1176310 T8752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1176311 T8752 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1176311 T8752 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1176312 T8752 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1176312 T8752 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1176312 T8752 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1176313 T8752 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1176313 T8752 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1176314 T8752 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1176318 T8752 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1176328 T8752 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176333 T8752 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1176333 T8752 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1176334 T8752 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=53109793, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1176335 T8752 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7e8846ae lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@76fe9833),segFN=segments_1,generation=1}
   [junit4]   2> 1176335 T8752 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1176336 T8752 oass.SolrIndexSearcher.<init> Opening Searcher@450dfb8b[collection1] main
   [junit4]   2> 1176340 T8753 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@450dfb8b[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1176342 T8752 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1176343 T8705 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1
   [junit4]   2> 1176343 T8705 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1176343 T8756 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:31999/nq_yr collection:collection1 shard:shard1
   [junit4]   2> 1176345 T8756 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1176367 T8756 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1176369 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176369 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176370 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176369 T8756 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1176370 T8756 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C984 name=collection1 org.apache.solr.core.SolrCore@293a46e9 url=https://127.0.0.1:31999/nq_yr/collection1 node=127.0.0.1:31999_nq_yr C984_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:31999/nq_yr, core=collection1, node_name=127.0.0.1:31999_nq_yr}
   [junit4]   2> 1176370 T8756 C984 P31999 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:31999/nq_yr/collection1/
   [junit4]   2> 1176371 T8756 C984 P31999 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1176371 T8756 C984 P31999 oasc.SyncStrategy.syncToMe https://127.0.0.1:31999/nq_yr/collection1/ has no replicas
   [junit4]   2> 1176371 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1176371 T8756 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:31999/nq_yr/collection1/ shard1
   [junit4]   2> 1176371 T8756 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1176374 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176375 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176375 T8751 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176375 T8738 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176379 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1176382 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176490 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176490 T8751 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176490 T8738 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176532 T8756 oasc.ZkController.register We are https://127.0.0.1:31999/nq_yr/collection1/ and leader is https://127.0.0.1:31999/nq_yr/collection1/
   [junit4]   2> 1176532 T8756 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:31999/nq_yr
   [junit4]   2> 1176532 T8756 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1176532 T8756 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1176532 T8756 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1176534 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176535 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176535 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176535 T8756 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1176536 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1176537 T8730 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:31999/nq_yr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31999_nq_yr",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1176549 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1176638 T8705 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1176639 T8705 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1176644 T8705 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 1176646 T8705 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:32002
   [junit4]   2> 1176648 T8705 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1176649 T8705 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1176649 T8705 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351
   [junit4]   2> 1176650 T8705 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351/'
   [junit4]   2> 1176652 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176652 T8751 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176652 T8738 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1176688 T8705 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351/solr.xml
   [junit4]   2> 1176769 T8705 oasc.CoreContainer.<init> New CoreContainer 1356392011
   [junit4]   2> 1176769 T8705 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351/]
   [junit4]   2> 1176771 T8705 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1176772 T8705 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1176772 T8705 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1176773 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1176773 T8705 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1176774 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1176774 T8705 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1176775 T8705 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1176775 T8705 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1176790 T8705 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1176791 T8705 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1176791 T8705 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1176792 T8705 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31986/solr
   [junit4]   2> 1176792 T8705 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1176794 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1176796 T8767 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4632305e name:ZooKeeperConnection Watcher:127.0.0.1:31986 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1176797 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1176800 T8705 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1176807 T8769 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b8422a3 name:ZooKeeperConnection Watcher:127.0.0.1:31986/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1176807 T8705 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1176817 T8705 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1177820 T8705 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32002_nq_yr
   [junit4]   2> 1177847 T8705 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32002_nq_yr
   [junit4]   2> 1177851 T8729 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1177851 T8751 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1177851 T8769 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1177851 T8738 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1177868 T8770 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1177868 T8770 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1177869 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1177869 T8770 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1177870 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1177870 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1177871 T8730 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1177872 T8730 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:32002/nq_yr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32002_nq_yr",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1177872 T8730 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1177872 T8730 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1177876 T8729 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1177978 T8751 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1177978 T8729 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1177978 T8769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1177978 T8738 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1178871 T8770 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1178871 T8770 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351/collection1
   [junit4]   2> 1178871 T8770 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1178872 T8770 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1178872 T8770 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1178873 T8770 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1178874 T8770 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351/collection1/'
   [junit4]   2> 1178875 T8770 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351/collection1/lib/README' to classloader
   [junit4]   2> 1178876 T8770 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351/collection1/lib/classes/' to classloader
   [junit4]   2> 1178938 T8770 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 1179005 T8770 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1179107 T8770 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1179117 T8770 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1179740 T8770 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1179741 T8770 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1179741 T8770 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1179755 T8770 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1179759 T8770 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1179774 T8770 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1179779 T8770 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1179784 T8770 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1179785 T8770 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1179785 T8770 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1179786 T8770 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1179787 T8770 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1179787 T8770 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1179787 T8770 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1179788 T8770 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392905318351/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty2/
   [junit4]   2> 1179788 T8770 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2fd0bf38
   [junit4]   2> 1179789 T8770 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty2
   [junit4]   2> 1179789 T8770 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty2/index/
   [junit4]   2> 1179790 T8770 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1179790 T8770 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392905311970/jetty2/index
   [junit4]   2> 1179791 T8770 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.190651796096072]
   [junit4]   2> 1179791 T8770 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@60092d4d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cd2d30d),segFN=segments_1,generation=1}
   [junit4]   2> 1179792 T8770 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1179795 T8770 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1179796 T8770 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1179796 T8770 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1179797 T8770 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1179797 T8770 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1179797 T8770 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1179798 T8770 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1179798 T8770 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1179798 T8770 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1179799 T8770 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1179800 T8770 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1179800 T8770 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1179800 T8770 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1179801 T8770 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1179801 T8770 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1179802 T8770 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1179811 T8770 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1179816 T8770 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1179817 T8770 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1179818 T8770 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=53109793, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1179818 T8770 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@60092d4d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cd2d30d),segFN=segments_1,generation=1}
   [junit4]   2> 1179818 T8770 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1179819 T8770 oass.SolrIndexSearcher.<init> Opening Searcher@417de42c[collection1] main
   [junit4]   2> 1179822 T8771 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@417de42c[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit

[...truncated too long message...]

nnection Watcher:127.0.0.1:60188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1418367 T11571 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1418367 T11571 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1418370 T11571 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1418382 T11571 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:60188_solr
   [junit4]   2> 1418382 T11571 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1418384 T11571 oasc.Overseer.start Overseer (id=91285458793463812-127.0.0.1:60188_solr-n_0000000000) starting
   [junit4]   2> 1418394 T11571 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1418397 T11571 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1418400 T11571 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1418402 T12086 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1418402 T12086 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 1418403 T12085 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1418404 T12085 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1418405 T12085 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=12 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"12",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 1418405 T12085 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8, shard9, shard10, shard11, shard12]
   [junit4]   2> 1418406 T12085 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1418408 T12084 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1418408 T12082 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1418408 T12076 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1418522 T12042 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue/qn-
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:240)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:237)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:237)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.createData(DistributedQueue.java:284)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.offer(DistributedQueue.java:271)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:121)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 1418522 T12042 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1418904 T11571 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1418921 T11571 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1418926 T12084 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1418926 T11571 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1418927 T12085 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1418929 T12076 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 1418929 T12082 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 1418932 T11571 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:60188 60188
   [junit4]   2> 1419107 T11571 oas.SolrTestCaseJ4.tearDown ###Ending testPlaceholders
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1419109 T11571 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 1419110 T11571 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 1422111 T11571 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene45, sim=DefaultSimilarity, locale=es_GT, timezone=Europe/Malta
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=1,free=250702208,total=516816896
   [junit4]   2> NOTE: All tests run in this JVM: [PreAnalyzedFieldTest, HardAutoCommitTest, TestCSVResponseWriter, NotRequiredUniqueKeyTest, TestComponentsName, DirectSolrSpellCheckerTest, TestDistributedSearch, SolrIndexConfigTest, TestSchemaResource, IndexBasedSpellCheckerTest, FieldAnalysisRequestHandlerTest, HighlighterTest, TestReload, SignatureUpdateProcessorFactoryTest, ConvertedLegacyTest, ShardRoutingCustomTest, TestRecovery, TestManagedSchemaFieldResource, DistributedSuggestComponentTest, FileUtilsTest, TestSchemaSimilarityResource, TestOmitPositions, TestPhraseSuggestions, TestCollationKeyRangeQueries, AlternateDirectoryTest, SolrCmdDistributorTest, XsltUpdateRequestHandlerTest, TestStressReorder, TestSweetSpotSimilarityFactory, TestHighlightDedupGrouping, CachingDirectoryFactoryTest, StatsComponentTest, TestSolrXml, SpellingQueryConverterTest, StandardRequestHandlerTest, TermVectorComponentDistributedTest, HdfsBasicDistributedZk2Test, TestClassNameShortening, FullSolrCloudDistribCmdsTest, BlockDirectoryTest, CoreContainerCoreInitFailuresTest, TestAtomicUpdateErrorCases, PathHierarchyTokenizerFactoryTest, TestShardHandlerFactory, SuggesterTSTTest, PingRequestHandlerTest, TestIndexingPerformance, OpenCloseCoreStressTest, TestFreeTextSuggestions, TestUtils, XmlUpdateRequestHandlerTest, TestCloudManagedSchema, SolrRequestParserTest, TestSolrDeletionPolicy2, SpellCheckComponentTest, TestQuerySenderListener, TestCursorMarkWithoutUniqueKey, NoCacheHeaderTest, TestCollationField, DocumentAnalysisRequestHandlerTest, CursorPagingTest, TestHashPartitioner, TestImplicitCoreProperties, RequiredFieldsTest, TestFuzzyAnalyzedSuggestions, TestAnalyzedSuggestions, CollectionsAPIDistributedZkTest, TestSolrCoreProperties, BadIndexSchemaTest, TestUpdate, TestRandomMergePolicy, TestDistribDocBasedVersion, SpellCheckCollatorTest, DistanceFunctionTest, BasicZkTest, TestFastLRUCache, PrimUtilsTest, DirectSolrConnectionTest, HdfsSyncSliceTest, LeaderElectionIntegrationTest, TestFastWriter, TestPartialUpdateDeduplication, TestCopyFieldCollectionResource, ChaosMonkeyNothingIsSafeTest, DirectUpdateHandlerTest, TestRandomFaceting, TestStressRecovery, BasicDistributedZkTest, BasicDistributedZk2Test, OverseerTest]
   [junit4] Completed on J1 in 26.23s, 8 tests, 1 error <<< FAILURES!

[...truncated 571 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:905: There were test failures: 362 suites, 1563 tests, 3 errors, 35 ignored (7 assumptions)

Total time: 78 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java6 - Build # 2313 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java6/2313/

3 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:56574/collection1, https://127.0.0.1:56581/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:56574/collection1, https://127.0.0.1:56581/collection1]
	at __randomizedtesting.SeedInfo.seed([B93E01EEAC6C8D9B:38D88FF6DB33EDA7]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:387)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:635)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.queryServer(AbstractFullDistribZkTestBase.java:1427)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:524)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:296)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:113)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: no servers hosting shard: 
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:302)
	... 50 more


REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard1 is not consistent.  Got 84 from https://127.0.0.1:32425/collection1lastClient and got 83 from https://127.0.0.1:55930/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 84 from https://127.0.0.1:32425/collection1lastClient and got 83 from https://127.0.0.1:55930/collection1
	at __randomizedtesting.SeedInfo.seed([B93E01EEAC6C8D9B:38D88FF6DB33EDA7]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1230)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1209)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:149)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:12079/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:12079/collection1
	at __randomizedtesting.SeedInfo.seed([B93E01EEAC6C8D9B:38D88FF6DB33EDA7]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.BaseDistributedSearchTestCase.add(BaseDistributedSearchTestCase.java:465)
	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:661)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:146)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:883)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:840)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:94)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 46 more




Build Log:
[...truncated 10135 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> 700576 T4305 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 700576 T4305 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 700582 T4305 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./solrtest-BasicDistributedZkTest-1392855620288
   [junit4]   2> 700584 T4305 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 700584 T4306 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 700685 T4305 oasc.ZkTestServer.run start zk server on port:63543
   [junit4]   2> 700686 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 700691 T4312 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@47772c1b name:ZooKeeperConnection Watcher:127.0.0.1:63543 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 700691 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 700692 T4305 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 700697 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 700705 T4314 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13cc8988 name:ZooKeeperConnection Watcher:127.0.0.1:63543/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 700705 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 700706 T4305 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 700710 T4305 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 700713 T4305 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 700723 T4305 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 700726 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 700727 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 700739 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 700740 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 700845 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 700845 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 700849 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 700849 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 700859 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 700860 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 700862 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 700863 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 700866 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 700867 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 700870 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 700870 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 700873 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 700874 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 700881 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 700882 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 700885 T4305 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 700886 T4305 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 700896 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 700906 T4316 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a47fcf3 name:ZooKeeperConnection Watcher:127.0.0.1:63543/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 700906 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 701199 T4305 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 701204 T4305 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 701207 T4305 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:63547
   [junit4]   2> 701210 T4305 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 701210 T4305 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 701211 T4305 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615
   [junit4]   2> 701212 T4305 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615/'
   [junit4]   2> 701251 T4305 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615/solr.xml
   [junit4]   2> 701332 T4305 oasc.CoreContainer.<init> New CoreContainer 873513828
   [junit4]   2> 701333 T4305 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615/]
   [junit4]   2> 701335 T4305 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 701335 T4305 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 701336 T4305 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 701336 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 701337 T4305 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 701337 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 701338 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 701338 T4305 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 701339 T4305 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 701353 T4305 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 701353 T4305 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 701354 T4305 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 701354 T4305 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63543/solr
   [junit4]   2> 701355 T4305 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 701356 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 701359 T4327 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e10d5a4 name:ZooKeeperConnection Watcher:127.0.0.1:63543 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 701360 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 701371 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 701373 T4329 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4948ec57 name:ZooKeeperConnection Watcher:127.0.0.1:63543/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 701373 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 701375 T4305 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 701385 T4305 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 701390 T4305 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 701392 T4305 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63547_
   [junit4]   2> 701393 T4305 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63547_
   [junit4]   2> 701403 T4305 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 701405 T4305 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 701417 T4305 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:63547_
   [junit4]   2> 701418 T4305 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 701420 T4305 oasc.Overseer.start Overseer (id=91282185931653124-127.0.0.1:63547_-n_0000000000) starting
   [junit4]   2> 701430 T4305 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 701441 T4331 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 701441 T4305 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 701442 T4331 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 701444 T4305 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 701446 T4305 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 701449 T4330 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 701454 T4332 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 701455 T4332 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 701456 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 701456 T4332 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 701457 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 701458 T4330 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63547",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63547_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 701458 T4330 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 701459 T4330 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 701461 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 701462 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 702457 T4332 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 702457 T4332 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615/collection1
   [junit4]   2> 702457 T4332 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 702458 T4332 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 702458 T4332 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 702459 T4332 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 702460 T4332 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615/collection1/'
   [junit4]   2> 702462 T4332 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615/collection1/lib/classes/' to classloader
   [junit4]   2> 702462 T4332 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615/collection1/lib/README' to classloader
   [junit4]   2> 702527 T4332 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 702597 T4332 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 702698 T4332 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 702708 T4332 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 703344 T4332 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 703344 T4332 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 703345 T4332 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 703357 T4332 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 703362 T4332 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 703375 T4332 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 703380 T4332 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 703385 T4332 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 703386 T4332 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 703386 T4332 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 703387 T4332 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 703388 T4332 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 703388 T4332 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 703388 T4332 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 703389 T4332 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392855620615/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/control/data/
   [junit4]   2> 703389 T4332 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e66da2c
   [junit4]   2> 703390 T4332 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/control/data
   [junit4]   2> 703390 T4332 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/control/data/index/
   [junit4]   2> 703391 T4332 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 703391 T4332 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/control/data/index
   [junit4]   2> 703392 T4332 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@596d2691
   [junit4]   2> 703393 T4332 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@4fd0a9bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@cff2775; maxCacheMB=0.6103515625 maxMergeSizeMB=0.06640625)),segFN=segments_1,generation=1}
   [junit4]   2> 703393 T4332 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 703396 T4332 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 703397 T4332 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 703397 T4332 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 703397 T4332 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 703398 T4332 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 703398 T4332 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 703399 T4332 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 703399 T4332 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 703399 T4332 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 703400 T4332 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 703400 T4332 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 703401 T4332 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 703401 T4332 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 703402 T4332 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 703402 T4332 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 703403 T4332 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 703412 T4332 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 703419 T4332 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 703419 T4332 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 703420 T4332 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 703420 T4332 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@4fd0a9bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@cff2775; maxCacheMB=0.6103515625 maxMergeSizeMB=0.06640625)),segFN=segments_1,generation=1}
   [junit4]   2> 703421 T4332 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 703421 T4332 oass.SolrIndexSearcher.<init> Opening Searcher@2ad1795c[collection1] main
   [junit4]   2> 703424 T4333 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2ad1795c[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 703425 T4332 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 703426 T4305 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1
   [junit4]   2> 703426 T4305 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 703426 T4336 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:63547 collection:control_collection shard:shard1
   [junit4]   2> 703428 T4336 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 703431 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 703434 T4338 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f861004 name:ZooKeeperConnection Watcher:127.0.0.1:63543/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 703434 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 703447 T4305 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 703458 T4336 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 703459 T4305 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 703461 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 703461 T4336 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 703462 T4336 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C826 name=collection1 org.apache.solr.core.SolrCore@61b50cc7 url=https://127.0.0.1:63547/collection1 node=127.0.0.1:63547_ C826_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:63547, core=collection1, node_name=127.0.0.1:63547_}
   [junit4]   2> 703462 T4336 C826 P63547 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:63547/collection1/
   [junit4]   2> 703462 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 703463 T4336 C826 P63547 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 703463 T4336 C826 P63547 oasc.SyncStrategy.syncToMe https://127.0.0.1:63547/collection1/ has no replicas
   [junit4]   2> 703463 T4336 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:63547/collection1/ shard1
   [junit4]   2> 703464 T4336 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 703466 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 703467 T4338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 703467 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 703472 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 703475 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 703608 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 703608 T4338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 703675 T4336 oasc.ZkController.register We are https://127.0.0.1:63547/collection1/ and leader is https://127.0.0.1:63547/collection1/
   [junit4]   2> 703675 T4336 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:63547
   [junit4]   2> 703675 T4336 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 703675 T4336 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 703676 T4336 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 703677 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 703677 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 703677 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 703677 T4336 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 703679 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 703680 T4330 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63547",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63547_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 703688 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 703754 T4305 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 703755 T4305 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 703760 T4305 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 703762 T4305 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:63551
   [junit4]   2> 703764 T4305 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 703765 T4305 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 703765 T4305 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164
   [junit4]   2> 703766 T4305 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164/'
   [junit4]   2> 703795 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 703795 T4338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 703805 T4305 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164/solr.xml
   [junit4]   2> 703888 T4305 oasc.CoreContainer.<init> New CoreContainer 809282123
   [junit4]   2> 703889 T4305 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164/]
   [junit4]   2> 703891 T4305 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 703892 T4305 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 703892 T4305 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 703893 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 703893 T4305 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 703894 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 703894 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 703895 T4305 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 703895 T4305 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 703907 T4305 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 703908 T4305 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 703908 T4305 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 703909 T4305 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63543/solr
   [junit4]   2> 703910 T4305 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 703911 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 703914 T4349 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a980dee name:ZooKeeperConnection Watcher:127.0.0.1:63543 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 703915 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 703918 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 703926 T4351 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2056eee name:ZooKeeperConnection Watcher:127.0.0.1:63543/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 703926 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 703932 T4305 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 704935 T4305 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63551_
   [junit4]   2> 704946 T4305 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63551_
   [junit4]   2> 704949 T4338 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 704949 T4351 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 704949 T4329 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 704967 T4352 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 704968 T4352 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 704969 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 704969 T4352 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 704969 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 704970 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 704971 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 704972 T4330 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63551",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63551_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 704972 T4330 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 704972 T4330 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 704975 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 705078 T4351 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 705078 T4338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 705078 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 705971 T4352 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 705971 T4352 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164/collection1
   [junit4]   2> 705971 T4352 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 705972 T4352 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 705972 T4352 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 705973 T4352 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 705974 T4352 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164/collection1/'
   [junit4]   2> 705975 T4352 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164/collection1/lib/README' to classloader
   [junit4]   2> 705976 T4352 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164/collection1/lib/classes/' to classloader
   [junit4]   2> 706040 T4352 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 706110 T4352 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 706211 T4352 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 706221 T4352 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 706861 T4352 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 706862 T4352 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 706863 T4352 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 706875 T4352 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 706880 T4352 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 706893 T4352 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 706898 T4352 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 706903 T4352 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 706904 T4352 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 706904 T4352 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 706905 T4352 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 706906 T4352 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 706906 T4352 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 706907 T4352 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 706907 T4352 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392855623164/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty1/
   [junit4]   2> 706907 T4352 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e66da2c
   [junit4]   2> 706908 T4352 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty1
   [junit4]   2> 706909 T4352 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty1/index/
   [junit4]   2> 706909 T4352 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 706909 T4352 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty1/index
   [junit4]   2> 706910 T4352 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@127a9665
   [junit4]   2> 706911 T4352 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@4cde8dd0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e27ca5d; maxCacheMB=0.6103515625 maxMergeSizeMB=0.06640625)),segFN=segments_1,generation=1}
   [junit4]   2> 706911 T4352 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 706915 T4352 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 706915 T4352 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 706916 T4352 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 706916 T4352 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 706916 T4352 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 706917 T4352 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 706917 T4352 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 706918 T4352 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 706918 T4352 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 706919 T4352 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 706919 T4352 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 706920 T4352 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 706920 T4352 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 706921 T4352 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 706921 T4352 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 706922 T4352 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 706932 T4352 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 706937 T4352 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 706937 T4352 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 706938 T4352 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 706939 T4352 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@4cde8dd0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e27ca5d; maxCacheMB=0.6103515625 maxMergeSizeMB=0.06640625)),segFN=segments_1,generation=1}
   [junit4]   2> 706939 T4352 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 706939 T4352 oass.SolrIndexSearcher.<init> Opening Searcher@556eee60[collection1] main
   [junit4]   2> 706943 T4353 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@556eee60[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 706946 T4352 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 706947 T4305 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1
   [junit4]   2> 706947 T4305 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 706947 T4356 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:63551 collection:collection1 shard:shard1
   [junit4]   2> 706949 T4356 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 706969 T4356 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 706971 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 706971 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 706972 T4356 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 706972 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 706972 T4356 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C827 name=collection1 org.apache.solr.core.SolrCore@3dca4d12 url=https://127.0.0.1:63551/collection1 node=127.0.0.1:63551_ C827_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:63551, core=collection1, node_name=127.0.0.1:63551_}
   [junit4]   2> 706973 T4356 C827 P63551 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:63551/collection1/
   [junit4]   2> 706973 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 706973 T4356 C827 P63551 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 706974 T4356 C827 P63551 oasc.SyncStrategy.syncToMe https://127.0.0.1:63551/collection1/ has no replicas
   [junit4]   2> 706974 T4356 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:63551/collection1/ shard1
   [junit4]   2> 706974 T4356 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 706976 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 706977 T4351 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 706977 T4338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 706977 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 706982 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 706985 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 707088 T4351 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 707088 T4338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 707088 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 707134 T4356 oasc.ZkController.register We are https://127.0.0.1:63551/collection1/ and leader is https://127.0.0.1:63551/collection1/
   [junit4]   2> 707134 T4356 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:63551
   [junit4]   2> 707134 T4356 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 707134 T4356 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 707135 T4356 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 707136 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 707136 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 707136 T4356 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 707137 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 707137 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 707138 T4330 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:63551",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63551_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 707148 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 707250 T4351 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 707250 T4338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 707250 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 707253 T4305 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 707254 T4305 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 707258 T4305 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1] of [SSLv2Hello, SSLv3, TLSv1]
   [junit4]   2> 707260 T4305 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:12079
   [junit4]   2> 707262 T4305 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 707263 T4305 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 707263 T4305 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655
   [junit4]   2> 707264 T4305 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655/'
   [junit4]   2> 707304 T4305 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655/solr.xml
   [junit4]   2> 707387 T4305 oasc.CoreContainer.<init> New CoreContainer 2138205711
   [junit4]   2> 707388 T4305 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655/]
   [junit4]   2> 707390 T4305 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 707390 T4305 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 707391 T4305 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 707391 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 707392 T4305 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 707392 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 707393 T4305 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 707393 T4305 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 707394 T4305 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 707408 T4305 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 707409 T4305 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 707409 T4305 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 707410 T4305 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63543/solr
   [junit4]   2> 707410 T4305 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 707411 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 707416 T4367 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@558de79b name:ZooKeeperConnection Watcher:127.0.0.1:63543 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 707417 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 707420 T4305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 707427 T4369 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7bea6fae name:ZooKeeperConnection Watcher:127.0.0.1:63543/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 707428 T4305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 707433 T4305 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 708437 T4305 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:12079_
   [junit4]   2> 708445 T4305 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:12079_
   [junit4]   2> 708449 T4351 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 708449 T4369 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 708449 T4338 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 708449 T4329 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 708467 T4370 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 708467 T4370 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 708469 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 708469 T4370 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 708469 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 708470 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 708470 T4330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 708471 T4330 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:12079",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:12079_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 708471 T4330 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 708472 T4330 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 708475 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 708579 T4351 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 708579 T4369 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 708579 T4338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 708579 T4329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 709470 T4370 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 709470 T4370 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655/collection1
   [junit4]   2> 709470 T4370 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 709471 T4370 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 709472 T4370 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 709473 T4370 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 709473 T4370 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655/collection1/'
   [junit4]   2> 709475 T4370 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655/collection1/lib/classes/' to classloader
   [junit4]   2> 709476 T4370 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655/collection1/lib/README' to classloader
   [junit4]   2> 709540 T4370 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 709610 T4370 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 709711 T4370 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 709721 T4370 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 710352 T4370 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 710353 T4370 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 710354 T4370 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 710366 T4370 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 710371 T4370 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 710384 T4370 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 710389 T4370 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 710394 T4370 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 710395 T4370 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 710395 T4370 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 710395 T4370 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 710396 T4370 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 710397 T4370 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 710397 T4370 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 710397 T4370 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392855626655/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty2/
   [junit4]   2> 710398 T4370 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e66da2c
   [junit4]   2> 710399 T4370 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty2
   [junit4]   2> 710399 T4370 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty2/index/
   [junit4]   2> 710399 T4370 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 710400 T4370 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392855620287/jetty2/index
   [junit4]   2> 710400 T4370 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@20f93c4c
   [junit4]   2> 710401 T4370 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@666a5680 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ae5b27d; maxCacheMB=0.6103515625 maxMergeSizeMB=0.06640625)),segFN=segments_1,generation=1}
   [junit4]   2> 710401 T4370 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 710405 T4370 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 710405 T4370 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 710406 T4370 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 710406 T4370 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 710406 T4370 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 710407 T4370 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 710407 T4370 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 710407 T4370 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 710408 T4370 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 710408 T4370 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 710409 T4370 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 710409 T4370 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 710410 T4370 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 710410 T4370 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 710411 T4370 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 710411 T4370 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 710420 T4370 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 710425 T4370 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 710426 T4370 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 710426 T4370 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 710427 T4370 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@666a5680 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ae5b27d; maxCacheMB=0.6103515625 maxMergeSizeMB=0.06640625)),segFN=segments_1,generation=1}
   [junit4]   2> 710427 T4370 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 710428 T4370 oass.SolrIndexSearcher.<init> Opening Searcher@6bd2ab50[collection1] main
   [junit4]   2> 710432 T4371 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6bd2ab50[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 710434 T4370 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 710435 T4305 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J1
   [junit4]   2> 710436 T4305 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 710435 T4374 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:12079 collection:collection1 shard:shard2
   [junit4]   2> 710437 T4374 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 710462 T4374 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 710464 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 710464 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 710464 T4374 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 710465 T4329 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 710465 T4374 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C828 name=collection1 org.apache.solr.core

[...truncated too long message...]

est-1392855840930/jetty3/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.BasicDistributedZk2Test-1392855840930/jetty3/index;done=false>>]
   [junit4]   2> 1012920 T12491 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.BasicDistributedZk2Test-1392855840930/jetty3/index
   [junit4]   2> 1012921 T12491 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election/91282200391712779-core_node3-n_0000000003
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:174)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:171)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:171)
   [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:67)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:180)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:399)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:188)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:399)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:259)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 1012922 T12491 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1012922 T12491 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1012922 T12491 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1012922 T12491 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Lucene46: {foo_b=MockVariableIntBlock(baseBlockSize=13), n_f1=PostingsFormat(name=MockSep), foo_d=PostingsFormat(name=SimpleText), foo_f=MockVariableIntBlock(baseBlockSize=13), n_tl1=PostingsFormat(name=MockSep), n_d1=PostingsFormat(name=Asserting), rnd_b=PostingsFormat(name=SimpleText), intDefault=PostingsFormat(name=Asserting), n_td1=PostingsFormat(name=MockSep), timestamp=PostingsFormat(name=Asserting), id=PostingsFormat(name=SimpleText), range_facet_sl=MockVariableIntBlock(baseBlockSize=13), range_facet_si=PostingsFormat(name=MockSep), oddField_s=MockVariableIntBlock(baseBlockSize=13), multiDefault=PostingsFormat(name=MockSep), n_tf1=PostingsFormat(name=Asserting), n_dt1=PostingsFormat(name=MockSep), range_facet_l=PostingsFormat(name=Asserting), n_ti1=PostingsFormat(name=SimpleText), text=MockVariableIntBlock(baseBlockSize=13), _version_=PostingsFormat(name=Asserting), SubjectTerms_mfacet=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=Asserting), n_tdt1=PostingsFormat(name=Asserting), other_tl1=PostingsFormat(name=Asserting), n_l1=PostingsFormat(name=Asserting), a_si=PostingsFormat(name=MockSep)}, docValues:{}, sim=DefaultSimilarity, locale=bg, timezone=Africa/Kinshasa
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=1,free=117622744,total=473366528
   [junit4]   2> NOTE: All tests run in this JVM: [TestImplicitCoreProperties, TestCollapseQParserPlugin, TestZkChroot, TestSolrQueryParserDefaultOperatorResource, TestCSVResponseWriter, TestCopyFieldCollectionResource, TestDistribDocBasedVersion, DocValuesMultiTest, TestReloadAndDeleteDocs, NoCacheHeaderTest, TestBinaryField, TestManagedSchemaFieldResource, TestSchemaSimilarityResource, MultiTermTest, SuggesterTSTTest, TestRangeQuery, TestDefaultSimilarityFactory, TestReload, CollectionsAPIDistributedZkTest, PreAnalyzedFieldTest, SpellingQueryConverterTest, BadIndexSchemaTest, LeaderElectionIntegrationTest, TestStressRecovery, TestFastOutputStream, TestFastWriter, TestUtils, TestCSVLoader, TestAnalyzedSuggestions, DebugComponentTest, TestFiltering, StatsComponentTest, HdfsBasicDistributedZk2Test, TestPhraseSuggestions, XmlUpdateRequestHandlerTest, TestQuerySenderListener, SignatureUpdateProcessorFactoryTest, SpellCheckComponentTest, SolrPluginUtilsTest, DisMaxRequestHandlerTest, TestStressUserVersions, TestDistributedMissingSort, DirectSolrSpellCheckerTest, HdfsUnloadDistributedZkTest, HdfsRecoveryZkTest, SortByFunctionTest, TestNonDefinedSimilarityFactory, StandardRequestHandlerTest, OpenCloseCoreStressTest, TestDistributedSearch, ConvertedLegacyTest, TestCollationKeyRangeQueries, TermVectorComponentDistributedTest, FieldAnalysisRequestHandlerTest, PluginInfoTest, PingRequestHandlerTest, OpenExchangeRatesOrgProviderTest, TestHighlightDedupGrouping, SolrIndexConfigTest, DistributedQueryComponentCustomSortTest, CachingDirectoryFactoryTest, TestFuzzyAnalyzedSuggestions, HighlighterConfigTest, DocumentAnalysisRequestHandlerTest, StressHdfsTest, TestSolrCoreProperties, MinimalSchemaTest, IndexBasedSpellCheckerTest, UniqFieldsUpdateProcessorFactoryTest, NumericFieldsTest, DistributedSuggestComponentTest, TestSchemaNameResource, TestSimpleQParserPlugin, SolrCmdDistributorTest, TestFieldResource, TestHashPartitioner, TestIndexingPerformance, TestSolrJ, TestArbitraryIndexDir, TestNumberUtils, TestFastLRUCache, DefaultValueUpdateProcessorTest, TestLazyCores, PreAnalyzedUpdateProcessorTest, TestQuerySenderNoQuery, TestAddFieldRealTimeGet, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test]
   [junit4] Completed on J1 in 92.74s, 1 test, 1 error <<< FAILURES!

[...truncated 667 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:905: There were test failures: 362 suites, 1563 tests, 2 errors, 1 failure, 35 ignored (7 assumptions)

Total time: 70 minutes 22 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure